├── .ebextensions ├── 01_python.config ├── 02_django.config └── alb-http-to-https-redirection.config ├── .gitignore ├── .python-version ├── .travis.yml ├── Pipfile ├── Pipfile.lock ├── README.asciidoc ├── Vagrantfile ├── ansible ├── pg_vars.yml ├── playbook.yml └── roles.yml ├── assets ├── GITenberg_repos_list_2.tsv ├── Git-logo.svg ├── Gitenberg_full.png ├── Gitenberg_full.svg ├── Gitenberg_full_white_stroke.png ├── Gitenberg_full_white_stroke.svg ├── copyright-symbols │ └── publicdomain.svg ├── covers │ ├── Adventures-of-Huckleberry-Finn_76.png │ ├── Adventures-of-Huckleberry-Finn_76_thumb.png │ ├── Don-Quixote_996.png │ ├── Don-Quixote_996_thumb.png │ ├── Frankenstein_84.png │ ├── Frankenstein_84_thumb.png │ ├── Jane-Eyre_1260.png │ ├── Jane-Eyre_1260_thumb.png │ ├── Moby-Dick--Or-The-Whale_2701.jpg │ ├── Moby-Dick--Or-The-Whale_2701_thumb.jpg │ ├── Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23.png │ ├── Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23_thumb.png │ ├── Pride-and-Prejudice_1342.png │ ├── Pride-and-Prejudice_1342_thumb.png │ ├── The-Adventures-of-Sherlock-Holmes_1661.png │ ├── The-Adventures-of-Sherlock-Holmes_1661_thumb.png │ ├── The-Brothers-Karamazov_28054.png │ ├── The-Brothers-Karamazov_28054_thumb.png │ ├── The-Time-Machine_35.png │ ├── The-Time-Machine_35_thumb.png │ ├── Twenty-Thousand-Leagues-under-the-Sea_164.png │ ├── Twenty-Thousand-Leagues-under-the-Sea_164_thumb.png │ ├── the-star-lord.jpg │ ├── the-star-lord_thumb.png │ ├── the-wailing-octopus.jpg │ └── the-wailing-octopus_thumb.png ├── css │ └── site.css ├── favicon.ico ├── gutenberg-logo-large.png └── gutenberg-logo-small.png ├── example.env ├── gitensite ├── __init__.py ├── apps │ ├── __init__.py │ ├── bookinfo │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── db.py │ │ ├── external.py │ │ ├── management │ │ │ ├── __init__.py │ │ │ └── commands │ │ │ │ ├── __init__.py │ │ │ │ ├── load_ids.py │ │ │ │ ├── load_repos.py │ │ │ │ ├── make_yaml.py │ │ │ │ └── write_yaml.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_auto_20160314_1933.py │ │ │ ├── 0003_auto_20180221_2032.py │ │ │ ├── 0004_auto_20180228_1951.py │ │ │ ├── 0005_auto_20180501_2256.py │ │ │ ├── 0006_auto_20200825_2051.py │ │ │ └── __init__.py │ │ ├── models.py │ │ └── views.py │ ├── bookrepos │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── forms.py │ │ ├── interfaces.py │ │ ├── management │ │ │ ├── __init__.py │ │ │ └── commands │ │ │ │ ├── __init__.py │ │ │ │ ├── make_github_token.py │ │ │ │ └── pull_repos.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_auto_20150315_2357.py │ │ │ ├── 0003_auto_20150316_0004.py │ │ │ ├── 0004_bookrepo_contributors.py │ │ │ ├── 0005_auto_20150325_0221.py │ │ │ ├── 0006_remove_bookrepo_repo_url.py │ │ │ ├── 0007_bookrepo_cover_url.py │ │ │ ├── 0008_auto_20150508_0015.py │ │ │ ├── 0009_auto_20150508_0048.py │ │ │ ├── 0010_githubauthtoken.py │ │ │ ├── 0011_bookrepo_etag.py │ │ │ ├── 0012_bookrepo_updated_at.py │ │ │ ├── 0013_ghcontributor.py │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── tests.py │ │ └── utils.py │ └── content │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── migrations │ │ └── __init__.py │ │ ├── models.py │ │ ├── templates │ │ ├── book_list.html │ │ ├── book_list_page.html │ │ ├── bookrepo_list.html │ │ ├── browsebooks.html │ │ ├── faq.html │ │ ├── footer.html │ │ ├── frame.html │ │ ├── get-involved.html │ │ ├── header.html │ │ ├── home.html │ │ ├── license.html │ │ ├── listing.html │ │ ├── newsletter.html │ │ ├── newsletters │ │ │ ├── 1.asciidoc │ │ │ ├── 1.html │ │ │ ├── 2.asciidoc │ │ │ ├── 2.html │ │ │ ├── 3.asciidoc │ │ │ ├── 3.html │ │ │ ├── 4.asciidoc │ │ │ ├── 4.html │ │ │ ├── 5.asciidoc │ │ │ └── 5.html │ │ ├── partials │ │ │ ├── featured-books.htmldjango │ │ │ ├── follow-us.htmldjango │ │ │ └── sign-up-announce.htmldjango │ │ └── updates.html │ │ ├── tests.py │ │ └── views.py ├── settings.py ├── urls.py └── wsgi.py ├── manage.py ├── requirements.txt ├── static └── .gitkeep └── upload └── null /.ebextensions/01_python.config: -------------------------------------------------------------------------------- 1 | packages: 2 | yum: 3 | git: [] 4 | cairo: [] 5 | cairo-devel: [] 6 | amazon-linux-extras: [] 7 | libffi.x86_64: [] 8 | libffi-devel: [] 9 | gcc: [] 10 | libjpeg-turbo-devel: [] 11 | 12 | commands: 13 | 01_postgres_activate: 14 | command: sudo amazon-linux-extras enable postgresql11 15 | 02_postgres_install: 16 | command: sudo yum install -y postgresql-devel 17 | 10_create_log_dir: 18 | command: "mkdir -p /var/log/django" 19 | 20_touch_log_file: 20 | command: "touch /var/log/django/django.log" 21 | 30_chown_log_file: 22 | command: "chmod 666 /var/log/django/django.log" 23 | 40_create_upload_dir: 24 | command: "mkdir -p /var/app/bundle/2/app/upload" 25 | 26 | 27 | container_commands: 28 | 02_migratedb: 29 | command: "source /var/app/venv/*/bin/activate && python manage.py migrate --noinput" 30 | leader_only: true 31 | ignoreErrors: true 32 | 03_collectstatic: 33 | command: "source /var/app/venv/*/bin/activate && python manage.py collectstatic --noinput" 34 | leader_only: true 35 | ignoreErrors: true 36 | -------------------------------------------------------------------------------- /.ebextensions/02_django.config: -------------------------------------------------------------------------------- 1 | option_settings: 2 | aws:elasticbeanstalk:container:python: 3 | WSGIPath: gitensite.wsgi:application 4 | -------------------------------------------------------------------------------- /.ebextensions/alb-http-to-https-redirection.config: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | #### Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | #### 4 | #### Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file 5 | #### except in compliance with the License. A copy of the License is located at 6 | #### 7 | #### http://aws.amazon.com/apache2.0/ 8 | #### 9 | #### or in the "license" file accompanying this file. This file is distributed on an "AS IS" 10 | #### BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 11 | #### License for the specific language governing permissions and limitations under the License. 12 | ################################################################################################### 13 | 14 | ################################################################################################### 15 | #### This configuration file modifies the default port 80 listener attached to an Application Load Balancer 16 | #### to automatically redirect incoming connections on HTTP to HTTPS. 17 | #### This will not work with an environment using the load balancer type Classic or Network. 18 | #### A prerequisite is that the 443 listener has already been created. 19 | #### Please use the below link for more information about creating an Application Load Balancer on 20 | #### the Elastic Beanstalk console. 21 | #### https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/environments-cfg-alb.html#environments-cfg-alb-console 22 | ################################################################################################### 23 | 24 | Resources: 25 | AWSEBV2LoadBalancerListener: 26 | Type: AWS::ElasticLoadBalancingV2::Listener 27 | Properties: 28 | LoadBalancerArn: 29 | Ref: AWSEBV2LoadBalancer 30 | Port: 80 31 | Protocol: HTTP 32 | DefaultActions: 33 | - Type: redirect 34 | RedirectConfig: 35 | Host: "#{host}" 36 | Path: "/#{path}" 37 | Port: "443" 38 | Protocol: "HTTPS" 39 | Query: "#{query}" 40 | StatusCode: "HTTP_301" 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Custom ignores 2 | venv/* 3 | *.swp 4 | *.ropeproject/* 5 | .idea 6 | al_notes.txt 7 | 8 | # Created by https://www.gitignore.io 9 | 10 | ### OSX ### 11 | .DS_Store 12 | .AppleDouble 13 | .LSOverride 14 | 15 | # Icon must end with two \r 16 | Icon 17 | 18 | 19 | # Thumbnails 20 | ._* 21 | upload/media* 22 | 23 | # Files that might appear on external disk 24 | .Spotlight-V100 25 | .Trashes 26 | 27 | # Directories potentially created on remote AFP share 28 | .AppleDB 29 | .AppleDesktop 30 | Network Trash Folder 31 | Temporary Items 32 | .apdisk 33 | 34 | 35 | ### Python ### 36 | # Byte-compiled / optimized / DLL files 37 | __pycache__/ 38 | *.py[cod] 39 | 40 | # C extensions 41 | *.so 42 | 43 | # Distribution / packaging 44 | .Python 45 | env/ 46 | build/ 47 | develop-eggs/ 48 | dist/ 49 | downloads/ 50 | eggs/ 51 | lib/ 52 | lib64/ 53 | parts/ 54 | sdist/ 55 | var/ 56 | *.egg-info/ 57 | .installed.cfg 58 | *.egg 59 | logs/ 60 | 61 | # PyInstaller 62 | # Usually these files are written by a python script from a template 63 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 64 | *.manifest 65 | *.spec 66 | 67 | # Installer logs 68 | pip-log.txt 69 | pip-delete-this-directory.txt 70 | 71 | # Unit test / coverage reports 72 | htmlcov/ 73 | .tox/ 74 | .coverage 75 | .cache 76 | nosetests.xml 77 | coverage.xml 78 | 79 | # Translations 80 | *.mo 81 | *.pot 82 | 83 | # Django stuff: 84 | *.log 85 | 86 | # Sphinx documentation 87 | docs/_build/ 88 | 89 | # PyBuilder 90 | target/ 91 | 92 | 93 | ### Django ### 94 | *.log 95 | *.pot 96 | *.pyc 97 | __pycache__/ 98 | local_settings.py 99 | 100 | .env 101 | db.sqlite3 102 | 103 | #pillow cache 104 | *static/ 105 | 106 | # Elastic Beanstalk Files 107 | .elasticbeanstalk/* 108 | !.elasticbeanstalk/*.cfg.yml 109 | !.elasticbeanstalk/*.global.yml 110 | 111 | # Ansible roles 112 | ansible/roles/* 113 | 114 | # Vagrant 115 | .vagrant 116 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.8.13 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - '3.6' 5 | 6 | env: 7 | - AWS_SECRET_ACCESS_KEY=abc DJANGO_SECRET_KEY=abc BOTO_CONFIG=/tmp 8 | 9 | before_install: 10 | - sudo mkdir /var/log/django 11 | - sudo chmod 777 /var/log/django 12 | 13 | install: pip install -r requirements.txt 14 | 15 | script: python manage.py test -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | gitberg = "==0.8.0" 10 | sorl-thumbnail = "==12.8.0" 11 | django-extensions = "==3.1.5" 12 | django-zurb-foundation = "==5.5.0" 13 | django-fontawesome = "==1.0" 14 | django-storages = "==1.12.3" 15 | django-el-pagination = "==3.3.0" 16 | django-debug-toolbar = "==3.2.4" 17 | dj-database-url = "==0.5.0" 18 | lxml = "==4.8.0" 19 | boto3 = "==1.21.40" 20 | appdirs = "==1.4.3" 21 | psycopg2 = "*" 22 | beautifulsoup4 = "==4.11.1" 23 | botocore = "==1.24.40" 24 | cairocffi = "==0.8.0" 25 | certifi = "==2021.10.8" 26 | cffi = "==1.15.0" 27 | charset-normalizer = "==2.0.12" 28 | cryptography = "==36.0.2" 29 | distlib = "==0.3.4" 30 | docopt = "==0.6.2" 31 | filelock = "==3.6.0" 32 | gitdb = "==4.0.9" 33 | "github3.py" = "==3.2.0" 34 | idna = "==3.3" 35 | isodate = "==0.6.1" 36 | jmespath = "==1.0.0" 37 | mock = "==2.0.0" 38 | msgpack = "==1.0.3" 39 | pbr = "==5.8.1" 40 | platformdirs = "==2.5.1" 41 | pluggy = "==1.0.0" 42 | psycopg2-binary = ">=2.8,<2.9" 43 | py = "==1.11.0" 44 | pycparser = "==2.21" 45 | pymarc = "==4.2.0" 46 | pyparsing = "==3.0.8" 47 | python-dateutil = "==2.8.2" 48 | pytz = "==2022.1" 49 | rdflib = "==6.1.1" 50 | requests = "==2.27.1" 51 | s3transfer = "==0.5.2" 52 | semver = "==2.2.0" 53 | sh = "==1.14.2" 54 | six = "==1.16.0" 55 | smmap = "==5.0.0" 56 | soupsieve = "==2.3.2.post1" 57 | sqlparse = "==0.4.2" 58 | toml = "==0.10.2" 59 | tox = "==3.25.0" 60 | uritemplate = "==4.1.1" 61 | urllib3 = "==1.26.9" 62 | virtualenv = "==20.14.1" 63 | wikipedia = "==1.4.0" 64 | Pillow = "==9.1.0" 65 | CacheControl = "==0.12.10" 66 | Django = "==2.2.15" 67 | GitPython = "==3.1.27" 68 | Jinja2 = "==3.1.1" 69 | MarkupSafe = "==2.1.1" 70 | PyJWT = "==2.3.0" 71 | pyOpenSSL = "==22.0.0" 72 | PyYAML = "==6.0" 73 | SPARQLWrapper = "==2.0.0" 74 | 75 | [requires] 76 | python_version = "3.8" 77 | -------------------------------------------------------------------------------- /README.asciidoc: -------------------------------------------------------------------------------- 1 | = A Django website for GITenberg 2 | 3 | The goal of gitensite is to: 4 | 5 | * provide information about the GITenberg project 6 | * document the major areas of work of GITenberg 7 | * connect visitor with areas to contribute 8 | * list information about GITenberg books 9 | * distribute completed ebooks 10 | 11 | == Set up 12 | This is a django/python project, so you are going to want to use some standard python dev tools. 13 | 14 | === environment 15 | Copy `example.env` to `.env` and store your local db secret and other values there. 16 | 17 | Virtual environments in python are a way of separating and reproducing the python package requirements. 18 | 19 | From the root of the project 20 | 21 | pipenv install -r requirements.txt 22 | 23 | For every new terminal, or when returning to work on the project, you will need to run 24 | 25 | source venv/bin/activate 26 | 27 | 28 | === Database(s) 29 | We use Postgres. 30 | 31 | To start a local db run: 32 | 33 | ./manage.py migrate 34 | ./manage.py createsuperuser 35 | 36 | If you have modified the database, generate new database migrations: 37 | 38 | ./manage.py makemigrations 39 | 40 | And run them with: 41 | 42 | ./manage.py migrate 43 | 44 | ==== Getting db data 45 | To fill out data in your local db, you'll first need to download and unpack the Project Gutenberg RDF metadata. you can run the provided `load_repos` management command: 46 | from https://www.gutenberg.org/cache/epub/feeds/rdf-files.tar.zip 47 | 48 | Then run 49 | ./manage.py load_repos 50 | 51 | Where `` is the address where you unzipped the rdf metadata 52 | 53 | 54 | == Apps 55 | 56 | There are currently three apps in the project: 57 | 58 | * _content_ for containing templates and views for website pages 59 | * _bookrepo_ [deprecated] for interacting with github and storing info about books 60 | * _bookinfo_ for metadata for the repos 61 | 62 | == Elastic Beanstalk 63 | this website is deployed on Elastic Beanstalk. Install the awsebcli tool in it's own py 3.7+ environment- it doesn't play well with other packages, even boto from aws! 64 | 65 | git push ssh://git-codecommit.us-east-1.amazonaws.com/v1/repos/giten_site 66 | 67 | cd to the project directory and you should be able to deploy updates using 68 | 69 | eb deploy [giten-site-new] 70 | 71 | Remember to commit changes before deploying. 72 | 73 | == Vagrant 74 | 75 | To install with Vagrant, follow these steps: 76 | 77 | . First install (on the local machine): 78 | ** VirtualBox 79 | ** Vagrant 80 | ** Ansible 81 | . Then from the project directory run this command: 82 | 83 | vagrant up 84 | 85 | . Then wait about 2 hours while it does its thing (loading the DB takes a very 86 | long time) 87 | . To log in: 88 | 89 | vagrant ssh 90 | 91 | . Once logged in 92 | 93 | cd giten_site 94 | + 95 | The virtualenv and environment variables should be automatically loaded upon login. 96 | 97 | . Start the server 98 | 99 | python manage.py runserver 0.0.0.0:5001 100 | + 101 | Use port 5001 since the Vagrantfile will automatically forward this port to the local port 5001. To use a different port, change the Vagrantfile on the host machine and run `vagrant halt` and `vagrant up`. Using the IP 0.0.0.0 will permit django to serve this on any IP. When running in vagrant the webserver doesn't recognize `localhost` addresses as being from localhost since it is port-forwarded into the virtual machine. 102 | 103 | . Go to http://localhost:5001 104 | . Done. 105 | 106 | == 2022 107 | 108 | updated to Amazon Linux 2 , python 3.8, and trying codecommit. Codecommit fights with github desktop so maybe should revert that. 109 | 110 | to update: 111 | 112 | giten_site % git commit -a -m 'message' 113 | 114 | # if this fails, go through `eb init` 115 | git push ssh://git-codecommit.us-east-1.amazonaws.com/v1/repos/giten_site 116 | 117 | eb deploy giten-site2-dev3 118 | 119 | # to see logs... 120 | eb logs giten-site2-dev3 121 | 122 | # to make a new environment 123 | 124 | eb create giten-site-something 125 | 126 | # to set environment vars 127 | 128 | eb setenv DJANGO_SECRET_KEY="????" GITENBERG_SECRET="????" AWS_SECRET_ACCESS_KEY="????" -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | 6 | config.vm.box = "ubuntu/trusty64" 7 | 8 | # Expose on the network 9 | config.vm.hostname = "gitensitedev.local" 10 | config.vm.network "forwarded_port", guest: 5001, host: 5001 # Dev site 11 | 12 | # Stolen from: 13 | # https://github.com/DSpace/vagrant-dspace/blob/master/Vagrantfile#L146 Turn 14 | # on SSH forwarding (so that 'vagrant ssh' has access to your local SSH keys, 15 | # and you can use your local SSH keys to access GitHub, etc.) 16 | config.ssh.forward_agent = true 17 | 18 | # provision the rest with ansible 19 | config.vm.provision "ansible" do |ansible| 20 | ansible.verbose = "v" 21 | ansible.playbook = "ansible/playbook.yml" 22 | ansible.galaxy_role_file = "ansible/roles.yml" 23 | end 24 | 25 | config.vm.provider "virtualbox" do |v| 26 | v.name = "gitensitedev" 27 | v.memory = 1024 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /ansible/pg_vars.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Basic settings 4 | postgresql_version: 9.4 5 | postgresql_encoding: 'UTF-8' 6 | postgresql_locale: 'en_US.UTF-8' 7 | 8 | postgresql_admin_user: "postgres" 9 | postgresql_default_auth_method: "trust" 10 | 11 | postgresql_databases: 12 | - name: gitensite 13 | owner: postgres 14 | 15 | postgresql_users: 16 | - name: postgres 17 | pass: gitensite 18 | encrypted: no 19 | -------------------------------------------------------------------------------- /ansible/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Provisioning a dev box 3 | # 4 | - name: Add Postgres Role 5 | become: yes 6 | become_user: root 7 | hosts: all 8 | roles: 9 | - ANXS.postgresql 10 | vars_files: 11 | - ./pg_vars.yml 12 | 13 | - name: Configuring Base System 14 | hosts: all 15 | tasks: 16 | - name: apt-get install stuff 17 | apt: pkg={{ item }} state=installed update_cache=true 18 | become: yes 19 | become_user: root 20 | with_items: 21 | - build-essential 22 | - git 23 | - libpq-dev 24 | - python 25 | - python-dev 26 | - python-pip 27 | - python-setuptools 28 | - shellcheck 29 | - unzip 30 | 31 | - name: Checkout gitensite repo 32 | git: repo=git@github.com:gitenberg-dev/giten_site.git dest=/home/vagrant/giten_site accept_hostkey=True 33 | 34 | - name: pip install stuff 35 | pip: name={{ item }} 36 | become: yes 37 | become_user: root 38 | with_items: 39 | - awscli 40 | - awsebcli 41 | - virtualenv 42 | 43 | - name: Virtualenv setup 44 | hosts: all 45 | tasks: 46 | - name: check if virtualenv already exists 47 | stat: path=/home/vagrant/venv 48 | register: venv_dir 49 | 50 | - name: create virtualenv for Django web app 51 | shell: virtualenv /home/vagrant/venv 52 | when: venv_dir.stat.isdir is not defined 53 | 54 | - name: install web application dependencies 55 | pip: requirements=/home/vagrant/giten_site/requirements.txt virtualenv=/home/vagrant/venv 56 | 57 | 58 | - name: Make logs 59 | become: yes 60 | become_user: root 61 | hosts: all 62 | tasks: 63 | - name: Make folder 64 | file: path=/var/log/django state=directory mode=0777 65 | 66 | - name: Make log file 67 | file: path=/var/log/django/django.log state=touch mode=777 68 | 69 | - name: Setup the database 70 | hosts: all 71 | environment: 72 | DJANGO_SECRET_KEY: STEVESTEVESTEVE 73 | ENVIRONMENT: DEVELOPMENT 74 | AWS_SECRET_ACCESS_KEY: STEVESTEVESTEVE 75 | DJANGO_DEBUG: yes 76 | tasks: 77 | - name: Django syncdb 78 | django_manage: command=syncdb app_path=/home/vagrant/giten_site virtualenv=/home/vagrant/venv 79 | 80 | - name: Django migrate 81 | django_manage: command=migrate app_path=/home/vagrant/giten_site virtualenv=/home/vagrant/venv 82 | 83 | - name: Django createsuperuser 84 | django_manage: command="createsuperuser --noinput --username=admin --email=admin@example.com" app_path=/home/vagrant/giten_site virtualenv=/home/vagrant/venv 85 | 86 | - name: check if rdf-files already exists 87 | stat: path=/home/vagrant/rdf-files 88 | register: rdf_dir 89 | 90 | - name: Make folder 91 | file: path=/home/vagrant/rdf-files state=directory mode=0776 92 | 93 | # Note: get_url ansible command fails for https 94 | - name: Download RDF dataset 95 | shell: "wget --quiet https://www.gutenberg.org/cache/epub/feeds/rdf-files.tar.zip -O/home/vagrant/rdf-files/rdf-files.tar.zip" 96 | when: rdf_dir.stat.isdir is not defined 97 | 98 | # Note: unarchive ansible command fails to unzip for some reason 99 | - name: Unzip and untar RDF dataset 100 | shell: "cd /home/vagrant/rdf-files && unzip rdf-files.tar.zip && tar -xf rdf-files.tar" 101 | when: rdf_dir.stat.isdir is not defined 102 | 103 | - name: Populate DB with data (takes a very long time) 104 | django_manage: command="load_repos /home/vagrant/rdf-files/cache/epub" app_path=/home/vagrant/giten_site virtualenv=/home/vagrant/venv 105 | when: rdf_dir.stat.isdir is not defined 106 | 107 | - name: Setup the environment 108 | hosts: all 109 | environment: 110 | DJANGO_SECRET_KEY: STEVESTEVESTEVE 111 | ENVIRONMENT: DEVELOPMENT 112 | AWS_SECRET_ACCESS_KEY: STEVESTEVESTEVE 113 | DJANGO_DEBUG: yes 114 | tasks: 115 | - name: create env file 116 | shell: "echo \"export DJANGO_SECRET_KEY=STEVESTEVESTEVE\nexport ENVIRONMENT=DEVELOPMENT\nexport AWS_SECRET_ACCESS_KEY=STEVESTEVESTEVE\nexport DJANGO_DEBUG=yes\" > /home/vagrant/giten_site/.env" 117 | 118 | - name: Add env setup to bashrc 119 | shell: "echo \"source /home/vagrant/venv/bin/activate\nsource /home/vagrant/giten_site/.env\" >> ~/.bashrc" 120 | -------------------------------------------------------------------------------- /ansible/roles.yml: -------------------------------------------------------------------------------- 1 | - src: ANXS.postgresql 2 | -------------------------------------------------------------------------------- /assets/Git-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /assets/Gitenberg_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/Gitenberg_full.png -------------------------------------------------------------------------------- /assets/Gitenberg_full.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml 86 | -------------------------------------------------------------------------------- /assets/Gitenberg_full_white_stroke.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/Gitenberg_full_white_stroke.png -------------------------------------------------------------------------------- /assets/Gitenberg_full_white_stroke.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 10 | 11 | 19 | 25 | 30 | 35 | 41 | 45 | 53 | 54 | 58 | 61 | 62 | -------------------------------------------------------------------------------- /assets/copyright-symbols/publicdomain.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 16 | 17 | -------------------------------------------------------------------------------- /assets/covers/Adventures-of-Huckleberry-Finn_76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Adventures-of-Huckleberry-Finn_76.png -------------------------------------------------------------------------------- /assets/covers/Adventures-of-Huckleberry-Finn_76_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Adventures-of-Huckleberry-Finn_76_thumb.png -------------------------------------------------------------------------------- /assets/covers/Don-Quixote_996.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Don-Quixote_996.png -------------------------------------------------------------------------------- /assets/covers/Don-Quixote_996_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Don-Quixote_996_thumb.png -------------------------------------------------------------------------------- /assets/covers/Frankenstein_84.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Frankenstein_84.png -------------------------------------------------------------------------------- /assets/covers/Frankenstein_84_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Frankenstein_84_thumb.png -------------------------------------------------------------------------------- /assets/covers/Jane-Eyre_1260.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Jane-Eyre_1260.png -------------------------------------------------------------------------------- /assets/covers/Jane-Eyre_1260_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Jane-Eyre_1260_thumb.png -------------------------------------------------------------------------------- /assets/covers/Moby-Dick--Or-The-Whale_2701.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Moby-Dick--Or-The-Whale_2701.jpg -------------------------------------------------------------------------------- /assets/covers/Moby-Dick--Or-The-Whale_2701_thumb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Moby-Dick--Or-The-Whale_2701_thumb.jpg -------------------------------------------------------------------------------- /assets/covers/Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23.png -------------------------------------------------------------------------------- /assets/covers/Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23_thumb.png -------------------------------------------------------------------------------- /assets/covers/Pride-and-Prejudice_1342.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Pride-and-Prejudice_1342.png -------------------------------------------------------------------------------- /assets/covers/Pride-and-Prejudice_1342_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Pride-and-Prejudice_1342_thumb.png -------------------------------------------------------------------------------- /assets/covers/The-Adventures-of-Sherlock-Holmes_1661.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/The-Adventures-of-Sherlock-Holmes_1661.png -------------------------------------------------------------------------------- /assets/covers/The-Adventures-of-Sherlock-Holmes_1661_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/The-Adventures-of-Sherlock-Holmes_1661_thumb.png -------------------------------------------------------------------------------- /assets/covers/The-Brothers-Karamazov_28054.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/The-Brothers-Karamazov_28054.png -------------------------------------------------------------------------------- /assets/covers/The-Brothers-Karamazov_28054_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/The-Brothers-Karamazov_28054_thumb.png -------------------------------------------------------------------------------- /assets/covers/The-Time-Machine_35.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/The-Time-Machine_35.png -------------------------------------------------------------------------------- /assets/covers/The-Time-Machine_35_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/The-Time-Machine_35_thumb.png -------------------------------------------------------------------------------- /assets/covers/Twenty-Thousand-Leagues-under-the-Sea_164.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Twenty-Thousand-Leagues-under-the-Sea_164.png -------------------------------------------------------------------------------- /assets/covers/Twenty-Thousand-Leagues-under-the-Sea_164_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/Twenty-Thousand-Leagues-under-the-Sea_164_thumb.png -------------------------------------------------------------------------------- /assets/covers/the-star-lord.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/the-star-lord.jpg -------------------------------------------------------------------------------- /assets/covers/the-star-lord_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/the-star-lord_thumb.png -------------------------------------------------------------------------------- /assets/covers/the-wailing-octopus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/the-wailing-octopus.jpg -------------------------------------------------------------------------------- /assets/covers/the-wailing-octopus_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/covers/the-wailing-octopus_thumb.png -------------------------------------------------------------------------------- /assets/css/site.css: -------------------------------------------------------------------------------- 1 | .logo-box{ 2 | margin: 3em 0; 3 | } 4 | 5 | .book-row { 6 | margin-bottom: 5px; 7 | } 8 | 9 | .centered { 10 | text-align: center; 11 | } 12 | 13 | .mission-text { 14 | text-align: center; 15 | line-height: 1.25; 16 | letter-spacing: .1em; 17 | } 18 | 19 | 20 | .blue { 21 | color: blue; 22 | } 23 | 24 | .reverse { 25 | display: inline-block; 26 | -moz-transform: scale(-1, 1); 27 | -webkit-transform: scale(-1, 1); 28 | transform: scale(-1, 1); 29 | } 30 | 31 | i.fa-magic { 32 | -webkit-transform:rotateY(180deg); 33 | -moz-transform:rotateY(180deg); 34 | -o-transform:rotateY(180deg); 35 | -ms-transform:rotateY(180deg); 36 | } 37 | 38 | .icon-box { 39 | text-align:center; 40 | color:#535554; 41 | } 42 | 43 | .text-center { 44 | text-align: center; 45 | } 46 | -------------------------------------------------------------------------------- /assets/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/favicon.ico -------------------------------------------------------------------------------- /assets/gutenberg-logo-large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/gutenberg-logo-large.png -------------------------------------------------------------------------------- /assets/gutenberg-logo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/assets/gutenberg-logo-small.png -------------------------------------------------------------------------------- /example.env: -------------------------------------------------------------------------------- 1 | ENVIRONMENT='DEVELOPMENT' 2 | DJANGO_SECRET_KEY='4pw*51yuo&are&-k(2k-!n3(@gs6vo^z8z)l&qt)xep)18c@3b' 3 | -------------------------------------------------------------------------------- /gitensite/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | -------------------------------------------------------------------------------- /gitensite/apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookinfo/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from .models import Book 3 | 4 | 5 | @admin.register(Book) 6 | class BookAdmin(admin.ModelAdmin): 7 | search_fields = ['title','book_id',] -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/db.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urljoin 2 | from django.core.files.base import ContentFile 3 | from django.core.files.storage import default_storage 4 | 5 | from gitenberg.metadata.pandata import Pandata 6 | 7 | from gitensite.apps.bookinfo.models import Author, Book, Cover 8 | 9 | import requests 10 | 11 | def addBookFromYaml(yaml): 12 | if isinstance(yaml, Pandata): 13 | obj = yaml.metadata 14 | else: 15 | pandata = Pandata() 16 | pandata.load(yaml) 17 | obj = pandata.metadata 18 | 19 | (book,created) = Book.objects.get_or_create(book_id=int(obj['identifiers']['gutenberg'])) 20 | 21 | if "_repo" in obj: 22 | book.repo_name = obj["_repo"] 23 | if "covers" in obj: 24 | #Delete any existing covers for this book 25 | Cover.objects.filter(book=book).delete() 26 | 27 | defaultCover = True 28 | for cover in obj["covers"]: 29 | #Upload cover to S3 30 | url = urljoin("https://raw.githubusercontent.com/GITenberg/" + obj["_repo"] + "/master/", cover["image_path"]) 31 | r = requests.get(url) 32 | if r.status_code == 200: 33 | contentfile = ContentFile(r.content) 34 | 35 | #Get image file extension from original filename 36 | if "." in cover["image_path"]: 37 | extension = cover["image_path"].split(".")[-1] 38 | else: 39 | extension = "png" 40 | 41 | uploadpath = obj['identifiers']['gutenberg'] + "." + extension 42 | 43 | #Add cover to database 44 | coverobject = Cover.objects.create( 45 | book=book, 46 | default_cover=defaultCover 47 | ) 48 | coverobject.file.save(uploadpath, contentfile) 49 | coverobject.file.close() 50 | 51 | #The first cover added will be the default cover 52 | defaultCover = False 53 | 54 | creator = None 55 | if "creator" in obj: 56 | creator = obj["creator"] 57 | elif "metadata" in obj and "creator" in obj.metadata: 58 | creator = obj.metadata["creator"] 59 | if creator is not None and "author" in creator: 60 | (author, created) = Author.objects.get_or_create(name=creator["author"]["agent_name"]) 61 | if "birthdate" in creator["author"]: 62 | author.birth_year = creator["author"]["birthdate"] 63 | if "deathdate" in creator["author"]: 64 | author.death_year = creator["author"]["deathdate"] 65 | book.author = author 66 | author.save() 67 | 68 | if "cover" in obj: 69 | num_existing_covers = len(list(Cover(book=book).objects.all())) 70 | (cover, created) = Cover.objects.get_or_create(link=obj["cover"]) 71 | cover.book = book 72 | cover.default_cover = (num_existing_covers == 0) 73 | 74 | book.title = obj["title"] 75 | book.language = obj["language"] if isinstance(obj["language"], str) else 'mul' 76 | 77 | if "description" in obj: 78 | book.description = obj["description"] 79 | if "gutenberg_type" in obj: 80 | book.gutenberg_type = obj["gutenberg_type"] 81 | elif "metadata" in obj and "gutenberg_type" in obj.metadata: 82 | book.gutenberg_type = obj.metadata["gutenberg_type"] 83 | 84 | bookshelf = None 85 | if "gutenberg_bookshelf" in obj: 86 | bookshelf = obj["gutenberg_bookshelf"] 87 | elif "metadata" in obj and "gutenberg_bookshelf" in obj.metadata: 88 | bookshelf = obj.metadata["gutenberg_bookshelf"] 89 | 90 | if bookshelf is not None: 91 | if type(bookshelf) is str: 92 | book.gutenberg_bookshelf = bookshelf 93 | else: 94 | book.gutenberg_bookshelf = ";".join(bookshelf) 95 | 96 | subjects = None 97 | if "subjects" in obj: 98 | subjects = obj["subjects"] 99 | elif "metadata" in obj and "subjects" in obj.metadata: 100 | subjects = obj.metadata["subjects"] 101 | 102 | if subjects is not None: 103 | if type(subjects) is str: 104 | book.subjects = subjects 105 | else: 106 | if len(subjects) > 0: 107 | if type(subjects[0]) is str: 108 | book.subjects = ";".join(subjects) 109 | else: 110 | subjectList = [x[1] for x in subjects] 111 | book.subjects = ";".join(subjectList) 112 | 113 | #yaml can either be a Pandata object or a YAML string, we need to handle either case 114 | if isinstance(yaml, Pandata): 115 | book.yaml = yaml.__unicode__ 116 | else: 117 | book.yaml = yaml 118 | 119 | book.save() 120 | 121 | return True -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/external.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from cachecontrol import CacheControl 3 | from lxml import etree 4 | 5 | request_session = requests.session() 6 | cached_session = CacheControl(request_session) 7 | 8 | #These functions take a Book object as an argument, and return either a URL if a result is found, or an empty string if there is no result 9 | 10 | def librivox_api(book): 11 | if not book.author: 12 | return "" 13 | if "," in book.author.name: 14 | author_last = book.author.name[0:book.author.name.index(",")] 15 | else: 16 | author_last = book.author.name 17 | 18 | result = cached_session.get("https://librivox.org/api/feed/audiobooks/", params={"author": author_last, "limit": 500, "format": "json"}) 19 | 20 | try: 21 | result_json = result.json() 22 | except: 23 | return "" 24 | 25 | if not "books" in result_json: 26 | return "" 27 | 28 | books = result_json["books"] 29 | for x in books: 30 | if "gutenberg.org" in x["url_text_source"] and x["url_text_source"].endswith("/" + str(book.book_id)): 31 | return x["url_librivox"] 32 | 33 | for x in books: 34 | if book.title == x["title"]: 35 | return x["url_librivox"] 36 | 37 | return "" 38 | 39 | def standard_ebooks_api(book): 40 | opds = cached_session.get("https://standardebooks.org/opds/all") 41 | 42 | try: 43 | parser = etree.XMLParser(recover=True) 44 | tree = etree.fromstring(opds.content, parser) 45 | except: 46 | return "" 47 | 48 | for entry in tree.iter("{http://www.w3.org/2005/Atom}entry"): 49 | for source in entry.iter("{http://purl.org/dc/elements/1.1/}source"): 50 | if "gutenberg" in source.text and source.text.endswith("/" + str(book.book_id)): 51 | return entry.find("{http://www.w3.org/2005/Atom}id").text 52 | 53 | return "" 54 | 55 | 56 | api_sources = {"Librivox": librivox_api, "Standard Ebooks": standard_ebooks_api} 57 | 58 | #This function calls each function in the api_sources dict, and returns a new dict containing the results 59 | def getExternalLinks(book): 60 | ret = {} 61 | 62 | for source in api_sources: 63 | result = api_sources[source](book) 64 | 65 | if len(result) > 0: 66 | ret[source] = result 67 | 68 | return ret 69 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookinfo/management/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookinfo/management/commands/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/management/commands/load_ids.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import csv 3 | 4 | from django.core.management.base import BaseCommand 5 | 6 | from gitensite.apps.bookinfo.models import Book 7 | 8 | class Command(BaseCommand): 9 | help = "load ids and repo names from Raymond's text TSV file" 10 | args = "" 11 | 12 | def handle(self, filename, *args, **kwargs): 13 | with open('/Users/eric/github/local/giten_site/assets/GITenberg_repos_list_2.tsv','r') as f: 14 | for vals in csv.reader(f,delimiter='\t', quotechar='"'): 15 | try: 16 | (book,created) = Book.objects.get_or_create(book_id=vals[1]) 17 | (book.repo_name, book.title, book.language) = (vals[2],vals[3],vals[4]) 18 | book.save() 19 | except (ValueError,IndexError): 20 | continue 21 | 22 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/management/commands/load_repos.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | # -*- coding: utf-8 -*- 3 | import csv 4 | import requests 5 | 6 | from django.core.management.base import BaseCommand 7 | from django.conf import settings 8 | 9 | from gitensite.apps.bookinfo.models import Author, Book, Cover, External_Link 10 | from gitenberg.util.catalog import BookMetadata, repo_list 11 | from gitenberg.config import NotConfigured 12 | 13 | from gitensite.apps.bookinfo.db import addBookFromYaml 14 | 15 | class Command(BaseCommand): 16 | help = "load ids and repo names from Raymond's text TSV file" 17 | args = "rdf_library", "start" 18 | def handle(self, rdf_library, start=0, *args, **kwargs): 19 | should_enrich = True 20 | if settings.DEBUG: 21 | should_enrich = False 22 | 23 | start = int(start) 24 | if start==0: 25 | Book.objects.all().delete() 26 | for (pg_id, repo_name) in repo_list: 27 | if int(pg_id)count: 29 | break 30 | if i%100 == 0: 31 | print(str(i)+" files completed") -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/management/commands/write_yaml.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.core.management.base import BaseCommand 4 | 5 | from gitensite.apps.bookinfo.models import Book 6 | 7 | import os 8 | 9 | class Command(BaseCommand): 10 | help = "write yaml files from db to ./metadata/.yaml" 11 | args = "" 12 | 13 | def handle(self, path, *args, **kwargs): 14 | for book in Book.objects.all(): 15 | with open(os.path.join(path,'{}.yaml'.format(book.book_id)), 'w') as _fp: 16 | _fp.write(book.yaml.encode("utf-8")) 17 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='Book', 15 | fields=[ 16 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 17 | ('book_id', models.IntegerField(unique=True)), 18 | ('repo_name', models.CharField(max_length=255, null=True, blank=True)), 19 | ('title', models.CharField(default=b'', max_length=1000, db_index=True)), 20 | ('language', models.CharField(default=b'en', max_length=5, db_index=True)), 21 | ('description', models.TextField(default=b'', null=True, db_index=True, blank=True)), 22 | ('yaml', models.TextField(default=b'', null=True)), 23 | ], 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/migrations/0002_auto_20160314_1933.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookinfo', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='book', 16 | name='description', 17 | field=models.TextField(default=b'', null=True, blank=True), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/migrations/0003_auto_20180221_2032.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookinfo', '0002_auto_20160314_1933'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='Author', 16 | fields=[ 17 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 18 | ('name', models.CharField(default=b'', max_length=255, null=True, blank=True)), 19 | ('aliases', models.CharField(default=b'', max_length=255, null=True, blank=True)), 20 | ('birth_year', models.IntegerField(null=True)), 21 | ('death_year', models.IntegerField(null=True)), 22 | ('wikipedia_url', models.URLField(max_length=500)), 23 | ], 24 | ), 25 | migrations.CreateModel( 26 | name='Cover', 27 | fields=[ 28 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 29 | ('link', models.URLField(max_length=500)), 30 | ('default_cover', models.BooleanField(default=False)), 31 | ], 32 | ), 33 | migrations.CreateModel( 34 | name='External_Link', 35 | fields=[ 36 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 37 | ('url', models.URLField(max_length=500)), 38 | ('source', models.CharField(max_length=255)), 39 | ], 40 | ), 41 | migrations.AddField( 42 | model_name='book', 43 | name='full_text', 44 | field=models.TextField(default=b'', null=True, blank=True), 45 | ), 46 | migrations.AddField( 47 | model_name='book', 48 | name='gutenberg_bookshelf', 49 | field=models.CharField(default=b'', max_length=1000), 50 | ), 51 | migrations.AddField( 52 | model_name='book', 53 | name='gutenberg_type', 54 | field=models.CharField(default=b'', max_length=255, null=True, blank=True), 55 | ), 56 | migrations.AddField( 57 | model_name='book', 58 | name='num_downloads', 59 | field=models.IntegerField(default=0), 60 | ), 61 | migrations.AddField( 62 | model_name='book', 63 | name='subjects', 64 | field=models.CharField(default=b'', max_length=1000), 65 | ), 66 | migrations.AlterField( 67 | model_name='book', 68 | name='book_id', 69 | field=models.IntegerField(), 70 | ), 71 | migrations.AddField( 72 | model_name='external_link', 73 | name='book', 74 | field=models.ForeignKey(to='bookinfo.Book', on_delete=models.CASCADE), 75 | ), 76 | migrations.AddField( 77 | model_name='cover', 78 | name='book', 79 | field=models.ForeignKey(to='bookinfo.Book', on_delete=models.CASCADE), 80 | ), 81 | migrations.AddField( 82 | model_name='book', 83 | name='author', 84 | field=models.ForeignKey(to='bookinfo.Author', on_delete=models.CASCADE, null=True), 85 | ), 86 | ] 87 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/migrations/0004_auto_20180228_1951.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookinfo', '0003_auto_20180221_2032'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='book', 16 | name='added', 17 | field=models.DateTimeField(auto_now_add=True, null=True), 18 | ), 19 | migrations.AddField( 20 | model_name='book', 21 | name='updated', 22 | field=models.DateTimeField(auto_now_add=True, null=True), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/migrations/0005_auto_20180501_2256.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookinfo', '0004_auto_20180228_1951'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RemoveField( 15 | model_name='cover', 16 | name='link', 17 | ), 18 | migrations.AddField( 19 | model_name='cover', 20 | name='file', 21 | field=models.FileField(null=True, upload_to=b'bookcovers/', blank=True), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/migrations/0006_auto_20200825_2051.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.27 on 2020-08-25 20:51 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('bookinfo', '0005_auto_20180501_2256'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='author', 17 | name='aliases', 18 | field=models.CharField(blank=True, default='', max_length=255, null=True), 19 | ), 20 | migrations.AlterField( 21 | model_name='author', 22 | name='name', 23 | field=models.CharField(blank=True, default='', max_length=255, null=True), 24 | ), 25 | migrations.AlterField( 26 | model_name='book', 27 | name='description', 28 | field=models.TextField(blank=True, default='', null=True), 29 | ), 30 | migrations.AlterField( 31 | model_name='book', 32 | name='full_text', 33 | field=models.TextField(blank=True, default='', null=True), 34 | ), 35 | migrations.AlterField( 36 | model_name='book', 37 | name='gutenberg_bookshelf', 38 | field=models.CharField(default='', max_length=1000), 39 | ), 40 | migrations.AlterField( 41 | model_name='book', 42 | name='gutenberg_type', 43 | field=models.CharField(blank=True, default='', max_length=255, null=True), 44 | ), 45 | migrations.AlterField( 46 | model_name='book', 47 | name='language', 48 | field=models.CharField(db_index=True, default='en', max_length=5), 49 | ), 50 | migrations.AlterField( 51 | model_name='book', 52 | name='subjects', 53 | field=models.CharField(default='', max_length=1000), 54 | ), 55 | migrations.AlterField( 56 | model_name='book', 57 | name='title', 58 | field=models.CharField(db_index=True, default='', max_length=1000), 59 | ), 60 | migrations.AlterField( 61 | model_name='book', 62 | name='yaml', 63 | field=models.TextField(default='', null=True), 64 | ), 65 | migrations.AlterField( 66 | model_name='cover', 67 | name='file', 68 | field=models.FileField(blank=True, null=True, upload_to='bookcovers/'), 69 | ), 70 | ] 71 | -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookinfo/migrations/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import logging 4 | 5 | from gitenberg.metadata.pandata import Pandata 6 | 7 | from django.db import models 8 | from django.utils import timezone as tz 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | gh_org = 'GITenberg' 13 | 14 | def smart_truncate(content, length=100, suffix='...'): 15 | if len(content) <= length: 16 | return content 17 | else: 18 | return ' '.join(content[:length+1].split(' ')[0:-1]) + suffix 19 | 20 | class Author(models.Model): 21 | name = models.CharField(max_length=255, default="", null=True, blank=True) 22 | aliases = models.CharField(max_length=255, default="", null=True, blank=True) 23 | birth_year = models.IntegerField(null=True) 24 | death_year = models.IntegerField(null=True) 25 | wikipedia_url = models.URLField(max_length=500) 26 | 27 | class Book(models.Model): 28 | book_id = models.IntegerField() 29 | repo_name = models.CharField(max_length=255, null=True, blank=True) 30 | title = models.CharField(max_length=1000, default="", db_index=True) 31 | language = models.CharField(max_length=5, default="en", null=False, db_index=True) 32 | description = models.TextField(default="", null=True, blank=True) 33 | author = models.ForeignKey(Author, on_delete=models.CASCADE, db_index=True, null=True) 34 | gutenberg_type = models.CharField(max_length=255, default="", null=True, blank=True) 35 | gutenberg_bookshelf = models.CharField(max_length=1000, default="") 36 | subjects = models.CharField(max_length=1000, default="") 37 | full_text = models.TextField(default="", null=True, blank=True) 38 | num_downloads = models.IntegerField(default=0) 39 | 40 | added = models.DateTimeField(auto_now_add=True, null=True) 41 | updated = models.DateTimeField(auto_now_add=True, null=True) 42 | 43 | yaml = models.TextField(null=True, default="") 44 | 45 | # using a custom save method in order to update the "updated" timestamp when specific fields are updated 46 | def save(self, *args, **kwargs): 47 | if self.pk: # if object already exists in db 48 | old_model = Book.objects.get(pk=self.pk) 49 | 50 | # This is the list of fields that, when modified, should update the "updated" timestamp 51 | fields = ["title", "language", "description", "author", "gutenberg_type", "gutenberg_bookshelf", "subjects", "full_text"] 52 | 53 | for field in fields: 54 | # If one of the fields was modified, update the timestamp 55 | if getattr(old_model, field, None) != getattr(self, field, None): 56 | self.updated = tz.now() 57 | super(Book, self).save(*args, **kwargs) # call the inherited save method 58 | 59 | def __unicode__(self): 60 | return self.repo_name 61 | 62 | @property 63 | def title_short(self): 64 | return smart_truncate(self.title, 65) 65 | 66 | @property 67 | def subjects_str(self): 68 | return self.subjects.replace(";", ", ") 69 | 70 | @property 71 | def author_first_last(self): 72 | if self.author is None: 73 | return None 74 | else: 75 | return " ".join(self.author.name.split(", ")[::-1]) 76 | 77 | @property 78 | def description_short(self): 79 | return smart_truncate(self.description, 300) 80 | 81 | @property 82 | def repo_url(self): 83 | return 'https://github.com/{}/{}'.format(gh_org,self.repo_name) 84 | 85 | @property 86 | def issues_url(self): 87 | return 'https://github.com/{}/{}/issues'.format(gh_org,self.repo_name) 88 | 89 | @property 90 | def downloads_url(self): 91 | return 'https://github.com/{}/{}/releases'.format(gh_org,self.repo_name) 92 | 93 | @property 94 | def pg_url(self): 95 | return 'https://www.gutenberg.org/ebooks/{}'.format(self.book_id) 96 | 97 | @property 98 | def cover_url(self): 99 | existing_covers = list(Cover.objects.filter(book=self)) 100 | for cover in existing_covers: 101 | if cover.default_cover and cover.file and hasattr(cover.file, "url"): 102 | return cover.file.url 103 | #No cover is set as default, so return the first cover that has a url 104 | if len(existing_covers) > 0: 105 | for cover in existing_covers: 106 | if cover.file and hasattr(cover.file, "url"): 107 | return cover.file.url 108 | return None 109 | 110 | _pandata=None 111 | def metadata(self): 112 | if not self._pandata: 113 | self._pandata=Pandata() 114 | self._pandata.load(self.yaml) 115 | return self._pandata.metadata 116 | 117 | class Cover(models.Model): 118 | book = models.ForeignKey(Book, on_delete=models.CASCADE, db_index=True) 119 | file = models.FileField(upload_to="bookcovers/", null=True, blank=True) 120 | default_cover = models.BooleanField(default=False) 121 | 122 | class External_Link(models.Model): 123 | book = models.ForeignKey(Book, on_delete=models.CASCADE, db_index=True) 124 | url = models.URLField(max_length=500) 125 | source = models.CharField(max_length=255) -------------------------------------------------------------------------------- /gitensite/apps/bookinfo/views.py: -------------------------------------------------------------------------------- 1 | # from django.shortcuts import render 2 | from django.http import HttpResponse,StreamingHttpResponse, JsonResponse 3 | from django.shortcuts import get_object_or_404 4 | from .models import Book 5 | 6 | def all_repos_txt(request): 7 | response = StreamingHttpResponse( 8 | ['{}\t{}\r'.format(book.book_id,book.repo_url) for book in Book.objects.all()], 9 | content_type="text/csv") 10 | return response 11 | 12 | def metadata(request, book_id, ext): 13 | book = get_object_or_404(Book, book_id=book_id ) 14 | if ext=='yaml': 15 | return HttpResponse(book.yaml,content_type="text/x-yaml; charset=utf-8") 16 | else: 17 | return JsonResponse(book.metadata()) 18 | 19 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookrepos/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from .models import BookRepo 3 | from .models import GitHubAuthToken 4 | from .models import GHContributor 5 | 6 | admin.site.register(BookRepo) 7 | admin.site.register(GitHubAuthToken) 8 | admin.site.register(GHContributor) 9 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/forms.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from django.forms import ModelForm 5 | from .models import BookRepo 6 | 7 | # class BookRepoForm(ModelForm): 8 | # class Meta: 9 | # model = BookRepo 10 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/interfaces.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import print_function 4 | import logging 5 | 6 | import github3 7 | 8 | from .models import BookRepo 9 | from .models import GHContributor 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | class GHContributorInterface(object): 15 | def __init__(self, book_repo, repo): 16 | """ Takes a BookRepo and a github repository obj 17 | """ 18 | self.book_repo = book_repo 19 | self.repo = repo 20 | 21 | def fulfill(self): 22 | for user in self.repo.contributors(): 23 | ghc = GHContributor() 24 | 25 | ghc.username = user.login 26 | ghc.book_repo = self.book_repo 27 | ghc.contributions = user.contributions 28 | 29 | ghc.save() 30 | 31 | class BookRepoInterface(object): 32 | 33 | def _get_book_id(self): 34 | """ get bookid out of repo.title """ 35 | name = self.repo.name 36 | try: 37 | return int(name.split('_')[-1]) 38 | except ValueError: 39 | return 40 | 41 | def _derive_contributor_string(self): 42 | """ Primative version stringification of contributors to the project """ 43 | # TODO: replace with a db model and a m2m fkey, BookRepo >< GHUsers 44 | contrib = [contrib.login for contrib in self.repo.contributors()] 45 | return ', '.join(contrib) 46 | 47 | def _do_fulfill(self): 48 | 49 | # properties of the same name in a github3.Repo & a BookRepo db model 50 | direct_set = ('clone_url', 'name', 'open_issues', 'html_url') 51 | for key in direct_set: 52 | setattr(self.book_repo, key, self.repo.__dict__[key]) 53 | 54 | self.book_repo.cover_url = 'http://placehold.it/140x200' # placeholder img 55 | self.book_repo.book_id = self._get_book_id() 56 | self.book_repo.save() 57 | 58 | # self.contributorint = GHContributorInterface(self.book_repo, self.repo) 59 | # self.contributorint.fulfill() 60 | # self.book_repo.contributors = self._derive_contributor_string() 61 | 62 | 63 | class GHSearchBookRepo(BookRepoInterface): 64 | """ """ 65 | def __init__(self, repo): 66 | super(GHSearchBookRepo, self).__init__() 67 | self.repo = repo 68 | self.book_id = self._get_book_id() 69 | self.book_repo, _ = BookRepo.objects.get_or_create(book_id=self.book_id) 70 | 71 | def fulfill(self): 72 | self._do_fulfill() 73 | return self.book_repo 74 | 75 | def __repr__(self): 76 | return u"Interface for {0} and {1}".format(self.book_repo, self.repo) 77 | 78 | 79 | class GithubToBookRepoInterface(BookRepoInterface): 80 | """ 81 | Takes a github3.Repository 82 | prepares a BookRepo model instance 83 | """ 84 | def __init__(self, repo): 85 | """ takes: a Github3.Repoitory """ 86 | self.repo = repo 87 | self.book_repo = BookRepo() 88 | 89 | def _get_bookid(self): 90 | """ get bookid out of repo.title """ 91 | name = self.repo.name 92 | try: 93 | return int(name.split('_')[-1]) 94 | except ValueError: 95 | return 96 | 97 | def fulfill(self): 98 | try: 99 | self._do_fulfill() 100 | return self.book_repo 101 | except github3.exceptions.ForbiddenError: 102 | print("ERROR: likely a 403 error") 103 | # TODO: python sleep for a while 104 | 105 | def _derive_contributor_string(self): 106 | """ Primative version stringification of contributors to the project """ 107 | # TODO: replace with a db model and a m2m fkey, BookRepo >< GHUsers 108 | contrib = [contrib.login for contrib in self.repo.contributors()] 109 | return ', '.join(contrib) 110 | 111 | def _do_fulfill(self): 112 | 113 | # properties of the same name in a github3.Repo & a BookRepo db model 114 | direct_set = ('clone_url', 'name', 'open_issues', 'html_url') 115 | for key in direct_set: 116 | setattr(self.book_repo, key, self.repo.__dict__[key]) 117 | 118 | self.book_repo.cover_url = 'http://placehold.it/140x200' # placeholder img 119 | self.book_repo.contributors = self._derive_contributor_string() 120 | self.book_repo.book_id = self._get_bookid() 121 | 122 | def __str__(self): 123 | return "A BookRepo: " + self.book_repo.name + str(self.book_repo.open_issues) 124 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookrepos/management/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookrepos/management/commands/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/management/commands/make_github_token.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | from getpass import getpass 4 | 5 | from django.core.management.base import BaseCommand 6 | from github3 import authorize 7 | 8 | from gitensite.apps.bookrepos.models import GitHubAuthToken 9 | 10 | 11 | class Command(BaseCommand): 12 | """ Authenticate a GitHub auth token """ 13 | 14 | def handle(self, *args, **kwargs): 15 | user = raw_input("GitHub username: ") 16 | password = '' 17 | 18 | while not password: 19 | password = getpass('Password for {0}: '.format(user)) 20 | 21 | note = 'gitenberg website github token auth' 22 | note_url = 'http://www.gitenberg.org' 23 | scopes = ['user', 'repo'] 24 | 25 | auth = authorize(user, password, scopes, note, note_url) 26 | auth_token = GitHubAuthToken(auth_id=auth.id, token=auth.token) 27 | auth_token.save() 28 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/management/commands/pull_repos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Django management command invoked with: 5 | ./manage.py pull_repos 6 | """ 7 | from __future__ import print_function 8 | import logging 9 | 10 | from django.core.management.base import BaseCommand 11 | 12 | from gitensite.apps.bookrepos.utils import SearchAllRepos 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | class Command(BaseCommand): 17 | """ 18 | """ 19 | args = "" 20 | help = """ 21 | Pulls info from book repositores on github 22 | belonging to the GITenberg organization. 23 | 24 | Takes: 25 | - string of github user, will ask for password to log in 26 | user doesn't have to be a member of the GITenberg org 27 | """ 28 | 29 | def handle(self, *args, **options): 30 | repo_gen = SearchAllRepos() 31 | repo_gen.run() 32 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='BookRepo', 15 | fields=[ 16 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 17 | ('book_id', models.IntegerField(blank=True)), 18 | ('title', models.CharField(max_length=255, blank=True)), 19 | ('url', models.URLField(max_length=255, blank=True)), 20 | ('repo_url', models.URLField(max_length=255, blank=True)), 21 | ], 22 | options={ 23 | }, 24 | bases=(models.Model,), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0002_auto_20150315_2357.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import datetime 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('bookrepos', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='Readme', 17 | fields=[ 18 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 19 | ('text', models.TextField()), 20 | ], 21 | options={ 22 | }, 23 | bases=(models.Model,), 24 | ), 25 | migrations.RenameField( 26 | model_name='bookrepo', 27 | old_name='title', 28 | new_name='name', 29 | ), 30 | migrations.AddField( 31 | model_name='bookrepo', 32 | name='clone_url', 33 | field=models.URLField(max_length=255, blank=True), 34 | preserve_default=True, 35 | ), 36 | migrations.AddField( 37 | model_name='bookrepo', 38 | name='created_at', 39 | field=models.DateTimeField(default=datetime.datetime.utcnow, auto_now_add=True), 40 | preserve_default=True, 41 | ), 42 | migrations.AddField( 43 | model_name='bookrepo', 44 | name='open_issues', 45 | field=models.IntegerField(default=0, blank=True), 46 | preserve_default=True, 47 | ), 48 | migrations.AlterField( 49 | model_name='bookrepo', 50 | name='book_id', 51 | field=models.IntegerField(unique=True, blank=True), 52 | preserve_default=True, 53 | ), 54 | ] 55 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0003_auto_20150316_0004.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0002_auto_20150315_2357'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='bookrepo', 16 | name='book_id', 17 | field=models.IntegerField(unique=True, null=True, blank=True), 18 | preserve_default=True, 19 | ), 20 | migrations.AlterField( 21 | model_name='bookrepo', 22 | name='clone_url', 23 | field=models.URLField(max_length=255, null=True, blank=True), 24 | preserve_default=True, 25 | ), 26 | migrations.AlterField( 27 | model_name='bookrepo', 28 | name='name', 29 | field=models.CharField(max_length=255, null=True, blank=True), 30 | preserve_default=True, 31 | ), 32 | migrations.AlterField( 33 | model_name='bookrepo', 34 | name='open_issues', 35 | field=models.IntegerField(default=0, null=True, blank=True), 36 | preserve_default=True, 37 | ), 38 | migrations.AlterField( 39 | model_name='bookrepo', 40 | name='repo_url', 41 | field=models.URLField(max_length=255, null=True, blank=True), 42 | preserve_default=True, 43 | ), 44 | migrations.AlterField( 45 | model_name='bookrepo', 46 | name='url', 47 | field=models.URLField(max_length=255, null=True, blank=True), 48 | preserve_default=True, 49 | ), 50 | ] 51 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0004_bookrepo_contributors.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0003_auto_20150316_0004'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='bookrepo', 16 | name='contributors', 17 | field=models.CharField(max_length=255, null=True, blank=True), 18 | preserve_default=True, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0005_auto_20150325_0221.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0004_bookrepo_contributors'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RenameField( 15 | model_name='bookrepo', 16 | old_name='url', 17 | new_name='html_url', 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0006_remove_bookrepo_repo_url.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0005_auto_20150325_0221'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RemoveField( 15 | model_name='bookrepo', 16 | name='repo_url', 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0007_bookrepo_cover_url.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0006_remove_bookrepo_repo_url'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='bookrepo', 16 | name='cover_url', 17 | field=models.URLField(max_length=255, null=True, blank=True), 18 | preserve_default=True, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0008_auto_20150508_0015.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import django.utils.timezone 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('bookrepos', '0007_bookrepo_cover_url'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='bookrepo', 17 | name='created_at', 18 | field=models.DateTimeField(default=django.utils.timezone.now, auto_now_add=True), 19 | preserve_default=True, 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0009_auto_20150508_0048.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0008_auto_20150508_0015'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='bookrepo', 16 | name='created_at', 17 | field=models.DateTimeField(auto_now_add=True), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0010_githubauthtoken.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0009_auto_20150508_0048'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='GitHubAuthToken', 16 | fields=[ 17 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 18 | ('token', models.TextField()), 19 | ('auth_id', models.TextField()), 20 | ('created_at', models.DateTimeField(auto_now_add=True)), 21 | ], 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0011_bookrepo_etag.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0010_githubauthtoken'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='bookrepo', 16 | name='etag', 17 | field=models.CharField(max_length=255, null=True, blank=True), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0012_bookrepo_updated_at.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import datetime 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('bookrepos', '0011_bookrepo_etag'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='bookrepo', 17 | name='updated_at', 18 | field=models.DateTimeField(default=datetime.datetime(2015, 6, 18, 22, 21, 0, 970969), auto_now=True), 19 | preserve_default=False, 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/0013_ghcontributor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('bookrepos', '0012_bookrepo_updated_at'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='GHContributor', 16 | fields=[ 17 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 18 | ('username', models.CharField(max_length=255, null=True, blank=True)), 19 | ('contributions', models.IntegerField(default=0, null=True, blank=True)), 20 | ('book_repo', models.ForeignKey(to='bookrepos.BookRepo', on_delete=models.CASCADE)), 21 | ], 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/bookrepos/migrations/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import base64 4 | import getpass 5 | import logging 6 | 7 | from django.db import models 8 | from github3 import login 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | class BookRepo(models.Model): 13 | book_id = models.IntegerField(null=True, blank=True, unique=True) 14 | name = models.CharField(max_length=255, null=True, blank=True) 15 | html_url = models.URLField(max_length=255, null=True, blank=True) 16 | # add: Contributors (fkey to GithubContributor model) 17 | clone_url = models.URLField(max_length=255, null=True, blank=True) 18 | open_issues = models.IntegerField(default=0, null=True, blank=True) 19 | contributors = models.CharField(max_length=255, null=True, blank=True) 20 | cover_url = models.URLField(max_length=255, null=True, blank=True) 21 | 22 | created_at = models.DateTimeField(auto_now_add=True) 23 | updated_at = models.DateTimeField(auto_now=True) 24 | etag = models.CharField(max_length=255, null=True, blank=True) 25 | 26 | def __unicode__(self): 27 | return self.name 28 | 29 | 30 | class GHContributor(models.Model): 31 | username = models.CharField(max_length=255, null=True, blank=True) 32 | book_repo = models.ForeignKey(BookRepo, on_delete=models.CASCADE) 33 | contributions = models.IntegerField(default=0, null=True, blank=True) 34 | 35 | def __unicode__(self): 36 | return "GHContributor {0} on {1}".format(self.username, self.book_repo.name) 37 | 38 | 39 | class Readme(models.Model): 40 | text = models.TextField() 41 | 42 | @classmethod 43 | def from_base64(self, content): 44 | text = base64.base64decode(content) 45 | return Readme(text=text) 46 | 47 | 48 | class GitHubAuthToken(models.Model): 49 | token = models.TextField() 50 | auth_id = models.TextField() 51 | 52 | created_at = models.DateTimeField(auto_now_add=True) 53 | 54 | @staticmethod 55 | def get_token_or_login(): 56 | # TODO: make the exception generate a token 57 | try: 58 | gh_token = GitHubAuthToken.objects.latest('created_at') 59 | gh = login(token=gh_token.token) 60 | logger.debug("Recovered gh token from db, used that to log in.") 61 | except GitHubAuthToken.DoesNotExist: 62 | logger.debug("GitHub Token not found, try running make_github_token") 63 | 64 | logger.info("Logging in to github but not saving token") 65 | gh = login(raw_input("username:"), password=getpass.getpass()) 66 | 67 | return gh 68 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /gitensite/apps/bookrepos/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import logging 4 | import time 5 | 6 | from .interfaces import GithubToBookRepoInterface 7 | from .interfaces import GHSearchBookRepo 8 | from .models import BookRepo 9 | from .models import GitHubAuthToken 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | class CommitTree(): 14 | """ 15 | Takes a github3.py Tree and provides interfaces 16 | http://github3py.readthedocs.org/en/latest/git.html#github3.git.Tree 17 | """ 18 | def __init__(self, tree): 19 | self.tree = tree 20 | self.dict = self.tree.as_dict() 21 | 22 | 23 | class SearchAllRepos(): 24 | def __init__(self): 25 | self.gh = GitHubAuthToken.get_token_or_login() 26 | self.check_ratelimit() 27 | 28 | def check_ratelimit(self): 29 | """ Check to see if we have enough ratelimits to continue, or if we 30 | should wait until timestamp 31 | Check on each run of run to see if we have >100 rate limits remaining 32 | if we do not, fetch the timestamp for when we can begin again """ 33 | # TODO: break if not enough ratelimits remaining 34 | self.rates = self.gh.rate_limit() 35 | logger.debug("\tGitHub api allocation {0}".format(self.rates['resources']['core']['remaining'])) 36 | logger.debug("\t{0} seconds until refresh".format( 37 | self.rates['resources']['core']['reset'] - 38 | int(time.time()) 39 | )) 40 | 41 | def run(self): 42 | for repo in self.gh.repositories_by('gitenberg'): 43 | interface = GHSearchBookRepo(repo) 44 | interface.fulfill() 45 | logger.debug(interface) 46 | 47 | 48 | class BookRepoGenerator(): 49 | def __init__(self, start=1, count=10): 50 | self.gh = GitHubAuthToken.get_token_or_login() 51 | self.check_ratelimit() 52 | 53 | def get_bookrepo(self, pg_id): 54 | """ takes `pg_id` a PG book id 55 | searches github for that id 56 | returns github repository object """ 57 | search_prefix = "user:GITenberg" 58 | search_id = "_{}".format(pg_id) 59 | search_str = " ".join([search_prefix, search_id]) 60 | 61 | result_iterator = self.gh.search_repositories(search_str) 62 | result = result_iterator.next() # fetching the first result from search 63 | logger.debug(result) 64 | return result.repository 65 | 66 | 67 | def run(self): 68 | for pg_id in self.todo: 69 | repo = self.get_bookrepo(pg_id) 70 | 71 | logger.debug("We have a repo from our iterator to digest") 72 | # Create an object that manages transfering data from a 73 | # Github3.Repo to a models.BookRepo 74 | interface = GithubToBookRepoInterface(repo) 75 | book_repo = interface.fulfill() 76 | logger.debug(interface) 77 | book_repo.save() 78 | 79 | self.run() 80 | 81 | 82 | def get_latest_etag(): 83 | # Not using this at the moment. 84 | latest_updated_book_repo = BookRepo.objects.order_by('-updated_at')[0] 85 | logger.debug("Latest updated book: {0}".format(latest_updated_book_repo)) 86 | latest_etag = latest_updated_book_repo.etag 87 | return latest_etag 88 | -------------------------------------------------------------------------------- /gitensite/apps/content/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/content/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/content/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /gitensite/apps/content/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/content/migrations/__init__.py -------------------------------------------------------------------------------- /gitensite/apps/content/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/book_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} 2 | {% block title %} GITenberg - Search for a book {% endblock title %} 3 | 4 | {% block body_content %} 5 | {% block body_js %} 6 | {% load staticfiles %} 7 | {{ block.super }} 8 | 9 | 10 | 11 | {% endblock %} 12 | 13 | {% include "header.html" %} 14 |
15 | {% include page_template %} 16 |
17 | 18 | {% endblock body_content %} 19 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/book_list_page.html: -------------------------------------------------------------------------------- 1 | {% load el_pagination_tags %} 2 | {% load staticfiles %} 3 | 4 | {% lazy_paginate object_list %} 5 | 6 | {% for book in object_list %} 7 |
8 |
9 | {% if book.cover_url %}{% else %}{% endif %} 10 |
11 |
12 |
{{ book.title_short }}
13 |

{{ book.author.name }}

14 |
15 |
16 |
17 |
18 |
{{book.description}}
19 |
20 |
Downloads 21 |
22 |
23 |
Repository 24 |
25 |
26 |
Issues 27 |
28 |
29 |
Gutenberg 30 |
31 |
32 |
33 |
34 |
35 |
36 | {% endfor %} 37 | {% show_more %} -------------------------------------------------------------------------------- /gitensite/apps/content/templates/bookrepo_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} 2 | 3 | {% block body_content %} 4 | 5 | {% include "header.html" %} 6 | 7 | {% for book in object_list %} 8 |
9 |
10 | 11 |
12 |
13 |
14 |

{{ book.name }}

15 | 16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 | 24 |
25 |
26 | clone url 27 |
28 |
29 | 30 |
31 |
32 | 33 |
34 | {{ book.open_issues }} Open Issues 35 |
36 |
37 | Contributors: {{ book.contributors }} 38 |
39 | 40 | 41 |
42 |
43 | {% endfor %} 44 | 45 | {% endblock body_content %} 46 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/browsebooks.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} {% load fontawesome %} {% load staticfiles %} {% block title %} GITenberg - Making beautiful books in the public domain {% endblock title %} {% block body_content %} 2 | 3 | {% include "header.html" %} 4 |
5 |
6 |

Popular Books

7 | 21 |

Recently Added

22 | 36 |

Recently Updated

37 | 51 |
52 | 53 | {% endblock body_content %} 54 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/faq.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} {% load fontawesome %} {% load staticfiles %} {% block title %} GITenberg - Making beautiful books in the public domain {% endblock title %} {% block body_content %} 2 | 3 |
4 |
5 |

Frequently Asked Questions

6 |

Why are we putting books on Github?

7 |

8 | In the world of modern software development, Github is the equivalent of the Library of Alexandria. 9 | It is a massive collection of software source code consisting of text. 10 | This source code is available as Free and Open Source Software (FOSS) with open licensing allowing developers and users to do what they wish with the source code. 11 |

12 |

13 | On a computer, books and source code look much the same: files with lines of text. 14 | We believe that the tools that have enabled collaboration on FOSS can can also be applied to the world of books. 15 |

16 |

17 | Repositories (which in our case, is a collection of text files with a history that represent a book and its metadata) can be forked. 18 | Forking means making a copy of a repository where you are free to make any changes you wish. 19 |

20 |

21 | More pragmatically, GitHub is free for public open source projects. 22 |

23 | 24 | 25 |

Who is this for?

26 |

27 | The early versions of GITenberg are going to be mainly for developers, 28 | and for people who know how to use git and GitHub. 29 | But the books that we curate and publish are for the benefit of all. 30 |

31 | 32 | 33 |

What can I do with these books?

34 |

35 | Anything you want, really! 36 | These books are in the public domain. 37 | You may quote them, edit them, print them, sell them, or give them away in whole or in part. 38 |

39 |

40 | See our license page for details. 41 |

42 |

43 | There are other nifty other projects that use public domain books. If you know of any (you may, as you're reading this), please let us know what they are so that we can link them here! 44 |

45 | 46 |

Can I sell these books?

47 |

48 | Yes, that is an acceptable use of works that are in the Public Domain. 49 |

50 |

51 | Note that not all of our covers are available for commercial use. 52 | See our license page for the specifics.. 53 |

54 | 55 |

How do I find books?

56 |

57 | We are working on a way to easily search for books in our collection. 58 | For now, the easiest way is to use the 59 | GitHub search 60 |

61 | 62 |

I found a problem in a book, who do I tell?

63 |
    64 |
  1. Find the book on github
    2. 65 |
  2. Click the "issues" tab
    3. 66 |
  3. Click "New Issue" (you may have to register for Github (and you should, github is awesome!))
    4. 67 |
  4. In the issue, copy and paste the sentence with an error as written so we can find the error in the book
    5. 68 |
  5. Describe what the text should say instead
    6. 69 |
70 | 71 |

72 | If you follow these steps, we (or you) should have enough information to fix the book. 73 |

74 | 75 | 76 | 77 |

Can I fix issues in books?

78 |

79 | Yes please!. 80 | This project is a collaboration between many people from all over the world. 81 | We are using the 82 | GitHub Flow 83 | workflow as our process for suggesting changes to books. 84 | These suggested changes are called Pull Requests. 85 |

86 |

87 | 88 |

89 | 90 |
91 |
92 | 93 | {% endblock body_content %} 94 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/footer.html: -------------------------------------------------------------------------------- 1 | 24 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/frame.html: -------------------------------------------------------------------------------- 1 | {% extends 'foundation/base.html' %} 2 | {% load fontawesome %} 3 | {% load staticfiles %} 4 | 5 | 6 | {% block meta %} 7 | 8 | 9 | {% endblock meta %} 10 | 11 | 12 | {# Custom CSS #} 13 | {% block css %} 14 | {% fontawesome_stylesheet %} 15 | 16 | 17 | {% endblock css %} 18 | 19 | 20 | {% block base_body %} 21 | 22 | 23 | 24 | {% block body_content %} 25 | {% endblock body_content %} 26 | 27 | 28 | {% include "footer.html" %} 29 | 30 | {% endblock base_body %} 31 | 32 | 33 | {% block body_js %} 34 | 44 | {% endblock body_js %} 45 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/get-involved.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} {% load fontawesome %} {% load staticfiles %} {% block title %} GITenberg - Making beautiful books in the public domain {% endblock title %} {% block body_content %} 2 | 3 | {% include "header.html" %} 4 | 5 |
6 |
7 |
8 | 9 |
10 |

Interested in contributing?

11 |
12 |

Awesome! For now there are a few things you can do depending on your interest and skill level.

13 |

14 | All of these things will require a Github account, so please make one. 15 |

16 |

17 | Firstly, if you find an error or typo in any of the books, report it in the 'Issues' tab on that repo. If you would like to offer changes: fork, edit and create a Pull Request. If you would like to make suggestions, help in another way, or would like to get more involved, you can join the project mailing list. 18 |

19 |
20 | 21 |
22 |

Helpful Github Tutorials

23 |
24 |

If you are new to Github, you may want to check out some of these tutorials to get started!

25 | 29 |

If you are using Windows, you will need to install Git for Windows.

30 |
31 |
32 |
33 |
34 | 35 |
36 |
37 |
38 | 39 |
40 |

Developers

41 |
42 |

43 | 53 |
54 | 55 |
56 |

Editors

57 |
58 | 68 |
69 | 70 |
71 |

Everyone Else

72 |
73 | 84 |
85 |
86 |
87 |
88 | {% endblock body_content %} 89 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/header.html: -------------------------------------------------------------------------------- 1 | {% load staticfiles %} 2 | 3 |
4 | 52 |
-------------------------------------------------------------------------------- /gitensite/apps/content/templates/home.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} 2 | {% load fontawesome %} 3 | {% load staticfiles %} 4 | {% load thumbnail %} 5 | 6 | {% block title %} GITenberg - Maintaining our Cultural Heritage in eBooks {% endblock title %} 7 | 8 | {% block body_content %} 9 | 10 | {% include "header.html" %} 11 | 12 |
13 |
14 | 15 |
16 |
17 | 18 |

Books

19 |
20 | 21 |
22 | 23 |
24 | 25 |
26 | 27 |

Git

28 |
29 | 30 |
31 |

32 | Gitenberg is a collaborative, 33 | open source community curating and publishing highly usable and attractive ebooks in the public domain. 34 | Our books are free to use by anyone for any purpose. 35 | They contain detailed metadata and are accessible in a wide variety of formats. 36 |

37 |
38 |
39 |
40 | 41 |
42 | 43 |
44 |
45 |
46 |
47 |
48 | 49 | 50 |

Find eBooks

51 | 52 |
53 |

Find free ebooks to download, read, improve, and savor.

54 |
55 | 56 |
57 |
58 |
59 | 60 | 61 | 62 |

Read the Docs

63 | 64 |
65 |

See the technical details and discussion surrounding gitenberg.

66 |
67 | 68 |
69 |
70 |
71 | 72 | 73 |

Get Involved

74 | 75 |
76 |

Learn more about contributing to the gitenberg project. All are welcome.

77 |
78 | 79 |
80 |
81 |
82 |
83 |
84 | 85 |
86 |
87 |
88 |

Popular books

89 |
90 |
91 | 105 |

View more...

106 |
107 | 108 |
109 |
110 |
111 | 112 |
113 |

Updates

114 |
115 |
116 |
Going into PG production.
117 |
118 |
    119 |
  • Live at PG
    • 120 |
  • More on tap
    • 121 |
122 |
123 |
Read More »
124 |
125 |
MILESTONE!
126 |
127 |
    128 |
  • Everything works!
    • 129 |
  • 57,000+ books rebuilt from source
    • 130 |
131 |
132 |
Read More »
133 |
134 |
Status Report #3
135 |
136 |
    137 |
  • Repos have been updated and cleaned
    • 138 |
  • We have a working asciidoc-to-epub building machine is operating
    • 139 |
  • This website!!!!
    • 140 |
141 |
142 |
Read More »
143 |
144 |
Status Report #2
145 |
146 |
    147 |
  • Ebook pipeline
    • 148 |
  • New website
    • 149 |
  • Upcoming events
    • 150 |
  • NYPL, DPLA, RTC and Whitehouse partnership
    • 151 |
152 |
153 |
Read More »
154 |
155 |
Status Report #1
156 |
157 |
    158 |
  • Knight Foundation Grant
    • 159 |
  • Next Steps
    • 160 |
  • Core Vision
    • 161 |
  • Book Formats
    • 162 |
  • Issues
    • 163 |
164 |
165 |
Read More »
166 |
167 |
168 | 169 |
170 |

Join Our Mailing List

171 |
172 |
173 |
Join the mailing list to get updates from the GITenberg community and the Free Ebook Foundation!
174 | {% include 'partials/sign-up-announce.htmldjango' %} 175 |
176 |
177 |
178 |
179 |
180 |
181 | {% endblock body_content %} 182 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/license.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} 2 | {% load fontawesome %} 3 | {% load staticfiles %} 4 | 5 | {% block title %} License -- GITenberg - Making beautiful books in the public domain {% endblock title %} 6 | 7 | {% block body_content %} 8 |
9 |
10 |

Copyright and Licensing

11 |

12 | All work by the GITenberg project is released under an open license. 13 |

14 |

15 | Our books in the public domain, and we assert no further copyright on them. 16 |

17 |

18 | Some of our books contain a cover image from the 19 | Recovering the Classics project from Creative Action Network. 20 | These covers are not in the public domain but are graciously available for use under the 21 | Creative Commons Attribution NonCommercial 4.0 License. 22 |

23 |

24 | Software created by the Gitenberg project is released under the 25 | GNU General Public License version 3. 26 |

27 | 28 |
29 |
30 | 31 | {% endblock body_content %} 32 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/listing.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} {% load fontawesome %} {% load staticfiles %} {% block title %} GITenberg - Making beautiful books in the public domain {% endblock title %} {% block body_content %} 2 | 3 | {% include "header.html" %} 4 |
5 |
6 |
7 |
8 | {% if book.cover_url %}{% else %}{% endif %} 9 |
10 |
11 |
12 |

{{ book.title }}

13 | {% if book.author %} 14 |

{{ book.author_first_last }}

15 | {% endif %} 16 |

Subjects: {{ book.subjects_str }}

17 |

Downloads: {{book.num_downloads }}

18 |
19 |
20 | Downloads
21 | Repository
22 | Issues
23 | Gutenberg 24 |
25 |
26 |
27 |
28 |
29 |

{{ book.description|default:"This book currently has no description. Feel free to open a pull request to add one!" }}

30 |
31 |
32 | 41 | {% if book.author %} 42 |
43 |
44 |

All Books by {{ book.author_first_last }}

45 |
46 |
47 |
48 |
49 | 50 |
51 |
52 |

53 |
54 |
55 | 56 |
57 |
58 |
59 | 73 |
74 | {% endif %} 75 | 76 | 159 | 160 | {% endblock body_content %} 161 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/newsletter.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/gitensite/apps/content/templates/newsletter.html -------------------------------------------------------------------------------- /gitensite/apps/content/templates/newsletters/1.asciidoc: -------------------------------------------------------------------------------- 1 | = GITenberg status report. 2 | Eric Hellman <> 3 | 4 | Seth started GITenberg back in September of 2012. It was pretty much a one person effort. Through this mailing list, a few other people started thinking about what it could be. I discovered the project and joined up in March of 2014 when I was exploring similar ideas. The project got some good exposure on Hacker News last August. 5 | 6 | 7 | == Knight Foundation Grant 8 | When I heard about the Knight News Challenge for Libraries, I suggested to Seth that GITenberg might be a good fit. Together with Raymond Yee, Seth and I put together a proposal. We got help from Jenny Lee, Phoebe Espiritu, and Emily Nimsakont. 9 | 10 | https://www.newschallenge.org/challenge/libraries/feedback/gitenberg-modern-maintenance-infrastructure-for-our-literary-heritage 11 | 12 | There were 676 entrants in the News Challenge, and believe it or not, GITenberg was one of 22 entries to receive funding. We've been awarded a $35,000 "Prototype Grant", which will allow us to spend some real development time to start turning the idea into something that really works. More to the point, we have a deadline (in late June!) for demonstrating the GITenberg concept. 13 | 14 | Now the work begins. 15 | 16 | == Next Steps 17 | 18 | Aside from 45,000+ repos on GitHub (a significant achievement by itself) GITenberg has so far been more concept than reality. If you tried to adopt a repo and submitted a pull request, you'll surely be aware that the GITenberg of today is more of a sketch than a working system. To make it a working system, we'll have to assemble a lot of cooperating components. Thankfully most of the components we need exist, and people are working on them. This became very clear at the Hack day sponsored by New York Public Library in January. 19 | 20 | So I think it's important to make that sketch more explicit. 21 | 22 | == Core Vision 23 | 24 | The core vision is that for any text in Project Gutenberg, anyone will be able to fork a repo, commit a change, and GITenberg machinery triggered by the commit will derive ebook files and metadata products. The commit can be submitted as a pull request, and accepted PRs will get fed back into Project Gutenberg. We hope. 25 | 26 | At this point, I should comment about Project Gutenberg. To fulfill its mission, Project Gutenberg has to be very conservative in its processes and operations. It doesn't have the resources to engage in speculative projects. So while the Project Gutenberg is enabling the experimentation we're doing, (and happy that we're doing it) we expect that GITenberg will need to prove itself before the PG feedback is a real thing. 27 | 28 | One thing that Project Gutenberg has been thinking about for years is the source format for its texts. For a good while, that format was 7 bit ascii text files, and there was a lot of resistance to migrating to anything more "modern". Now, the plain text you get from Project Gutenberg is utf-8. Sort of. The html files are maintained separately, and are not uniform; there's a lot of hand-coding. Changing the source format to RST, XML or TEI has been discussed. The PG ebook files (MOBI and EPUB) are built using a script called ebookmaker which digests the html files. The HTML files are thus the "source" files as far as the ebooks are concerned. It should be possible for us to duplicate this workflow in the GITenberg machinery. 29 | 30 | On the metadata side the situation is more obscure, and we're still working to understand it. There's a set of RDF files, there are metadata records associated with each ebook folder. 31 | 32 | 33 | == Book Formats 34 | 35 | We've surveyed the components now available, and we feel that we can also improve on the existing workflow by migrating away from HTML as a source format. At this point, asciidoc appears to be the best fit for a format that can be a source format for the required product files, while at the same time fitting with the established PG text corpus and the Git-based version control. It looks like the best choice for ebook and web formats is the HTMLBook flavor of HTML5. http://oreillymedia.github.io/HTMLBook/ There’s a converter for asciidoc that makes htmlbook files. https://github.com/oreillymedia/asciidoctor-htmlbook and css themes that support htmlbook. We expect that alternate paths into HTMLBook can be developed (or already exist) for LaTeX and TEI source formats. Pandoc has done quite a lot. 36 | 37 | Internet Archive seems like the best destination for GITenberg produced ebook files. 38 | 39 | NYPL Labs has done some really nice work on generating covers for PG texts, we expect to integrate that work as well. 40 | 41 | On the metadata side, we've started looking at YAML as an appropriate serialization for PG-associated metadata. conversion to MARC and other formats should be straightforward in the backend. 42 | 43 | == Issues 44 | 45 | Github itself has presented us with a set of challenges to address. The large number of repos in the GITenberg organization breaks some Github tools. For example, GitHub for Mac became unstable for me, and some 3rd party integrations would time out when we tried enabling them. We broke our Github pages. So we need to understand this better; Github support has been very responsive. There's a separate organization "gitenberg-dev" https://github.com/gitenberg-dev that we're using to let us easily work on code untill we fully understand how to work with 50,000 repos; at this point, you probably don’t want to be a member of the Gitenberg organization but you might want to join gitenberg-dev, even if you’re not a developer. 46 | 47 | The non-programmer usability of Github is another problem. We're going to set up a "github for poets" sandbox to see if this challenge can be addressed. 48 | 49 | Despite the Knight grant, and the efforts of some committed volunteers, this is still a very small effort. GITenberg can't succeed without a lot of help, cooperation, and collaboration. I hope everyone on this list will be help us nurture that success. 50 | 51 | Here’s something each of us can do to get the ball rolling: Decide on a Gitenberg repo to contribute to. Star it in Github. Then add it to the list of active repos at https://github.com/gitenberg-dev/documentation/blob/master/activerepos.csv 52 | (send a PR or create an issue https://github.com/gitenberg-dev/documentation/issues ) 53 | 54 | If you’re new to Github, instructions are at https://github.com/gitenberg-dev/documentation/blob/add_how_to/how_to.md 55 | 56 | There's a huge amount that we don't know, 57 | and so much prior work we've yet to absorb but we're really encouraged by all the expressions of support we've received. 58 | Thank you all! 59 | 60 | Eric 61 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/newsletters/2.asciidoc: -------------------------------------------------------------------------------- 1 | == GITenberg Status Report #2 2 | 3 | Quite a bit of work has occurred since our last status report, though it's rather scattered work in progress and still needs to be put together and documented. 4 | 5 | * We have 14 PG texts converted to asciidoc 6 | * We have a working asciidoc-to-epub build machine 7 | * We have the start of a django website 8 | * We'll be at http://www.bookexpoamerica.com/[Book Expo America] 9 | * We'll be participating in a hackathon in SFO in June 10 | 11 | In addition, our distribution partners were highlighted by President Obama in his ConnectED! 12 | 13 | 14 | === Ebook pipeline 15 | Conversions to asciidoc have been done using Pandoc + retouched by hand. epub building is done using Travis CI, and is triggered by a "tag"(so the epub files only get built when a version is declared, in analogy with the way software distributions are bundled and distributed.) So for example, the release files for the "Brothers Karamazov" repo: 16 | https://github.com/sethwoodworth/The-Brothers-Karamazov_28054 17 | are made available at https://github.com/sethwoodworth/The-Brothers-Karamazov_28054/releases/tag/v0.0.3 18 | The travis config is at 19 | https://github.com/sethwoodworth/The-Brothers-Karamazov_28054/blob/master/.travis.yml 20 | 21 | We're very thankful to the folks at Travis-CI who have gone the extra mile to get their systems working with the large numbers of repos in the GITenberg organization. 22 | 23 | Note that this is just proof of concept and does not reflect any decisions about repo layout, for example. The list of releases so far is here: 24 | https://github.com/sethwoodworth/nyplcodex/blob/master/final.asciidoc 25 | 26 | Obviously, this work will get migrated to the gitenberg-dev organization once it's assembled properly. 27 | 28 | Thanks to Nessie, and Sam Wilson for conversions to asciidoc; Raymond and Seth have done the work on the build system. 29 | 30 | === New website 31 | Al has been working on the website- it will provide search and linking to the corpus as well as a gateway into the documentation. His work is at https://github.com/gitenberg-dev/giten_site/ Feel free to contribute. It's running on http://www.gitenberg.org/ but please don't publicize it yet. 32 | 33 | === Upcoming events 34 | We'll be presenting at BookExpo America on May 27th at New York City's Javits Center as part of the BookExpo Challenge. Please come by and say hello! http://www.bookexpochallenge.com/ 35 | 36 | We're among the organizers in the CODEXHackathon. June 27 and 28 in San Francisco, to coincide with the American Library Association Annual Conference. Plans are still coming together on that. 37 | 38 | 39 | === NYPL, DPLA, RTC and Whitehouse partnership 40 | 41 | Buried in the details of President Obama's ConnectED announcement on April 30th was https://www.whitehouse.gov/the-press-office/2015/04/30/fact-sheet-spreading-joy-reading-more-children-and-young-adults[this tidbit]: 42 | 43 | * The Digital Public Library of America: Their network of librarians will volunteer with the New York Public Library to help make sure popular books reach the most appropriate audience. DPLA, in conjunction with Recovering the Classics are also add age-appropriate public domain titles whose text and cover art has been redesigned by leading graphic designers and artists. 44 | 45 | We've been working with http://shop.thecreativeactionnetwork.com/collections/recovering-the-classics[_Recovering the Classics_], which draws on the Creative Action Network's community of thousands of artists and designers, to build these public domain ebooks! 46 | 47 | === Next steps 48 | Progress has been slower in our metadata workflow development and in our community building work, but overall I think we've made some good progress, though there much work left to be done. Thank you for all your comments and support. -------------------------------------------------------------------------------- /gitensite/apps/content/templates/newsletters/3.asciidoc: -------------------------------------------------------------------------------- 1 | == GITenberg Status Report #3 2 | 3 | At our last report, most of the pieces were in place, but putting them all together, not so much. Since then, progress was very slow, as team members had to shift around responsibilities. 4 | 5 | * Repos have been updated and cleaned. 6 | * We have a working asciidoc-to-epub building machine is operating 7 | * This website!!!! 8 | 9 | In addition, our distribution partners were highlighted by President Obama in his ConnectED! 10 | 11 | 12 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/newsletters/3.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | GITenberg Status Report #3 9 | 10 | 411 | 412 | 413 | 415 |
416 |
417 |

GITenberg Status Report #3

418 |
419 |
420 |

At our last report, most of the pieces were in place, but putting them all together, not so much. Since then, progress was very slow, as team members had to shift around responsibilities.

421 |
422 |
423 |
    424 |
  • 425 |

    Repos have been updated and cleaned.

    426 |
    • 427 |
  • 428 |

    We have a working asciidoc-to-epub building machine is operating

    429 |
    • 430 |
  • 431 |

    This website!!!!

    432 |
    • 433 |
434 |
435 |
436 |

In addition, our distribution partners were highlighted by President Obama in his ConnectED!

437 |
438 |
439 |

Repo updates

440 | 441 |
442 |
443 |

The build system

444 | 445 |
446 |
447 |

Distribution

448 | 449 |
450 |
451 |

Website!!!

452 | 453 |
454 |
455 |
456 |
457 | 462 | 463 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/newsletters/4.asciidoc: -------------------------------------------------------------------------------- 1 | == GITenberg Status Report #4 2 | 3 | We've reached a big milestone for the GITenberg Project, which comes after a lot of work over 6 years by several groups of people. 4 | 5 | GITenberg is a prototype that explores how Project Gutenberg might work if all the Gutenberg texts were on Github, so that tools like version control, continuous integration, and pull-request workflow tools could be put to work. We hope that Project Gutenberg can take advantage of what we've learned; work in that direction has begun but needs resources and volunteers. Go check it out! 6 | 7 | 8 | It's hard to believe, but GITenberg started 6 years ago when Seth Woodworth started making Github repos for Gutenberg texts. I joined the project two years later when I started doing the same and discovered that Seth was 43,000 repos ahead of me. The project got a big boost when the Knight Foundation awarded us a Prototype Fund grant to explore the applicability of open-source methodologies to the maintenance of the cultural heritage that is the Project Gutenberg collection. But there were big chunks of effort left to finish the work when that grant ended. Last year, 6 computer-science seniors from Stevens Institute of Technology took up the challenge and brought the project within sight of a major milestone (if not the finishing-line). There remained only the reprocessing of 58,000 ebooks (with more being created every day!). As of last week, we've done that! Whew. 9 | 10 | 11 | So here's what's been done: 12 | 13 | * Almost 57,000 ebooks from Project Gutenberg have been loaded into Github repositories. 14 | * EPUB, PDF, and Kindle Ebooks have been rebuilt and added to releases for all but about 100 of these. 15 | * Github webhooks trigger dockerized ebook building machines running on AWS Elastic Beanstock every time a git repo is tagged. 16 | * Toolchains for asciidoc, HTML and plain text source files are running on the ebook builders. 17 | * A website at https://www.gitenberg.org/ uses the webhooks to index and link to all of the ebooks. 18 | www.gitenberg.org presents links to Github, Project Gutenberg, Librivox, and Standard Ebooks. 19 | * Cover images are supplied for every ebook. 20 | * Human-readable metadata files are available for every ebook 21 | * Syndication feeds for these books are made available in ONIX, MARC and OPDS via Unglue.it. 22 | 23 | Everything in this project is built in the hope that the bits can be incorporated into Project Gutenberg wherever appropriate. In January 2019, the US public domain will resume the addition of new books, so it's more important than ever that we strengthen the infrastructure that supports it. 24 | 25 | Some details: 26 | 27 | * All of the software that's been used is open source and content is openly licensed. 28 | * PG's epubmaker software has been significantly strengthened and improved. 29 | * About 200 PG ebooks have had fatal formatting errors remediated to allow for automated ebook file production. 30 | * 1,363 PG ebooks were omitted from this work due to licensing or because they aren't really books. 31 | * PG's RDF metadata files were converted to human-readable YAML and enhanced with data from New York Public Library and from Wikipedia. 32 | * Github API throttling limits the build/release rate to about 600 ebooks/hour/login. A full build takes about 4 full days with one github login. 33 | 34 | Acknowledgements: 35 | 36 | * Seth Woodworth. In retrospect, the core idea was obvious, audacious, and crazy. Like all great ideas. 37 | * Github tech support. Always responsive. 38 | * The O'Reilly HTMLBook team. The asciidoc toolchain is based on their work. 39 | * Plympton. Many asciidoc versions were contributed to GITenberg as part of the "Recovering the Classics" project. Thanks to Jenny 8. Lee, Michelle Cheng, Max Pevner and Nessie Fox. 40 | * Albert Carter and Paul Moss contributed to early versions of the GITeneberg website. 41 | * The Knight Foundation provided funding for GITenberg at a key juncture in the project's development though its prototype fund. The Knight Foundation supports public-benefitting innovation in so many ways even beyond the funding it provides, and we thank them with all our hearts. 42 | * Travis-CI. The first version of automated ebook building took advantage of Travis-CI. Thanks! 43 | * Raymond Yee got the automated ebook building to actually work. 44 | * New York Public Library contributed descriptions, rights info, and generative covers. They also sponsored hackathons that significantly advanced the environment for public domain books. Special thanks to Leonard Richardson, Mauricio Giraldo and Jens Troeger (Bookalope). 45 | * The Board at the Free Ebook Foundation: Seth, Vicky Reich, Rupert Gatti, Todd Carpenter, Michael Wolfe and Karen Liu. 46 | * The Stevens GITenberg team: Marc Gotliboym, Nicholas Tang-Mifsud, Brian Silverman, Brandon Rothweiler, Meng Qiu, and Ankur Ramesh. They redesigned the gitenberg.org website, added search, added automatic metadata updates, and built the dockerized elastic beanstalk ebook-builder and queuing system. This work was done as part of their two-semester capstone (project) course. The course is taught by Prof. David Klappholz, who managed a total of 23 student projects last academic year. 47 | * Last, but certainly not least, Greg Newby (Project Gutenberg) for consistent encouragement and tolerance of our nit-discovery, Juliet Sutherland (Distributed Proofreaders) for her invaluable insights into how PG ebooks get made, and to the countless volunteers at both organizations who collectively have made possible the preservation and reuse of our public domain. 48 | 49 | I'm sure I've omitted an important acknowledgement or two - please let me know so I can rectify the omission. -------------------------------------------------------------------------------- /gitensite/apps/content/templates/newsletters/5.asciidoc: -------------------------------------------------------------------------------- 1 | == GITenberg Status Report #5 2 | 3 | We're pleased to report that the GITenberg effort is helping https://www.gutenberg.org[Project Gutenberg] adopt some of the innovations piloted here. 4 | 5 | So far... 6 | 7 | * The bugfixes we made to the PG EbookMaker software have migrated into the production https://github.com/guenbergtools/EbookMaker[EbookMaker software] on Project Gutenberg. 8 | * Almost 200 problematic books identified in the GITenberg work are being rehabilitated on Project Gutenberg. 9 | * The generated covers used for GITenberg were adapted for Project Gutenberg. 10 | * Metadata processing software is being applied to generation of offline versions of Project Gutenberg. 11 | 12 | The following are being seriously considered for implementation at Project Gutenberg (resources permitting) ... 13 | 14 | * Use of Github for Project Gutenberg maintenance. 15 | * YAML metadata for Project Gutenberg books. 16 | 17 | If you have ideas for other ways to reuse the work done for this project, be sure to let us know! -------------------------------------------------------------------------------- /gitensite/apps/content/templates/partials/featured-books.htmldjango: -------------------------------------------------------------------------------- 1 | {% load staticfiles %} 2 | {% load thumbnail %} 3 |
4 |
5 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/Adventures-of-Huckleberry-Finn_76.png' '80' as im %} 6 | 7 | {% endthumbnail %} 8 |
9 | 10 |
11 |
The Adventures of Huckleberry Finn
12 |

by Twain, Mark

13 |
14 |
15 | 16 |
17 |
18 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/Don-Quixote_996.png' '80' as im %} 19 | 20 | {% endthumbnail %} 21 |
22 | 23 |
24 |
Don Quixote
25 |

by Cervantes Saavedra, Miguel de Ormsby

26 |
27 |
28 | 29 |
30 |
31 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/Frankenstein_84.png' '80' as im %} 32 | 33 | {% endthumbnail %} 34 |
35 | 36 |
37 |
Frankenstein
38 |

by Shelley, Mary Wollstonecraft

39 |
40 |
41 | 42 |
43 |
44 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/Jane-Eyre_1260.png' '80' as im %} 45 | 46 | {% endthumbnail %} 47 |
48 | 49 |
50 |
Jane Eyre
51 |

by Brontë, Charlotte

52 |
53 |
54 | 55 | 56 |
57 |
58 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23.png' '80' as im %} 59 | 60 | {% endthumbnail %} 61 |
62 | 63 |
64 |
The Narrative of the Life of Frederick Douglass an American Slave
65 |

by Douglass, Frederick

66 |
67 |
68 | 69 |
70 |
71 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/Pride-and-Prejudice_1342.png' '80' as im %} 72 | 73 | {% endthumbnail %} 74 |
75 | 76 |
77 |
Pride and Prejudice
78 |

by Austen, Jane

79 |
80 |
81 | 82 |
83 |
84 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/The-Adventures-of-Sherlock-Holmes_1661.png' '80' as im %} 85 | 86 | {% endthumbnail %} 87 |
88 | 89 |
90 |
The Adventures of Sherlock Holmes
91 |

by Doyle, Arthur Conan

92 |
93 |
94 | 95 |
96 |
97 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/The-Brothers-Karamazov_28054.png' '80' as im %} 98 | 99 | {% endthumbnail %} 100 |
101 | 102 |
103 |
The Brothers Karamazov
104 |

by Dostoyevsky, Fyodor

105 |
106 |
107 | 108 |
109 |
110 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/The-Time-Machine_35.png' '80' as im %} 111 | 112 | {% endthumbnail %} 113 |
114 | 115 |
116 |
The Time Machine
117 |

by Wells, H. G. (Herbert George)

118 |
119 |
120 | 121 |
122 |
123 | {% thumbnail request.scheme|add:'://'|add:request.get_host|add:STATIC_URL|add:'covers/Twenty-Thousand-Leagues-under-the-Sea_164.png' '80' as im %} 124 | 125 | {% endthumbnail %} 126 |
127 | 128 |
129 |
Twenty Thousand Leagues under the Sea
130 |

by Verne, Jules

131 |
132 |
-------------------------------------------------------------------------------- /gitensite/apps/content/templates/partials/follow-us.htmldjango: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 |

5 | 6 | 7 |

8 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/partials/sign-up-announce.htmldjango: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 5 |
6 |
7 |
8 |
9 | 10 |
11 |
12 | 13 | view archives 14 | 15 |
16 |
17 |
18 | 19 | -------------------------------------------------------------------------------- /gitensite/apps/content/templates/updates.html: -------------------------------------------------------------------------------- 1 | {% extends 'frame.html' %} {% load fontawesome %} {% load staticfiles %} {% block title %} GITenberg - Making beautiful books in the public domain {% endblock title %} {% block body_content %} 2 | 3 | {% include "header.html" %} 4 | 5 | 6 | 14 | 15 | 22 | 23 |
24 |
25 | 26 | 27 | 28 | 29 | 30 |
31 |
32 | 33 | {% endblock body_content %} 34 | -------------------------------------------------------------------------------- /gitensite/apps/content/tests.py: -------------------------------------------------------------------------------- 1 | # invoke with ./manage.py test gitensite.apps.content or ./manage.py test 2 | 3 | from django.test import TestCase 4 | from django.test.client import Client 5 | 6 | from gitensite.apps.bookinfo.models import Author, Book, Cover 7 | from gitensite.apps.bookinfo.external import getExternalLinks 8 | 9 | import os 10 | 11 | class PageTests(TestCase): 12 | 13 | def setUp(self): 14 | self.client = Client() 15 | 16 | def test_view_by_anonymous(self): 17 | anon_client = Client() 18 | r = anon_client.get("/", follow=True) 19 | self.assertEqual(r.status_code, 200) 20 | r = anon_client.get("/updates/", follow=True) 21 | self.assertEqual(r.status_code, 200) 22 | r = anon_client.get("/books/", follow=True) 23 | self.assertEqual(r.status_code, 200) 24 | r = anon_client.get("/search?q=moby+dick", follow=True) 25 | self.assertEqual(r.status_code, 200) 26 | r = anon_client.get("/get-involved/", follow=True) 27 | self.assertEqual(r.status_code, 200) 28 | r = anon_client.get("/faq/", follow=True) 29 | self.assertEqual(r.status_code, 200) 30 | r = anon_client.get("/license/", follow=True) 31 | self.assertEqual(r.status_code, 200) 32 | 33 | yaml = """_repo: Alice-s-Adventures-in-Wonderland_11 34 | alternative_title: Alice in Wonderland 35 | creator: 36 | author: 37 | agent_name: Carroll, Lewis 38 | alias: Dodgson, Charles Lutwidge 39 | birthdate: 1832 40 | deathdate: 1898 41 | gutenberg_agent_id: 7 42 | url: http://www.gutenberg.org/2009/agents/7 43 | wikipedia: http://en.wikipedia.org/wiki/Lewis_Carroll 44 | description: "See also our HTML edition: #928" 45 | gutenberg_bookshelf: Children's Literature 46 | gutenberg_issued: 2008-06-27 47 | gutenberg_type: Text 48 | identifiers: 49 | gutenberg: 11 50 | language: en 51 | publisher: Project Gutenberg 52 | rights: Public domain in the USA. 53 | rights_url: http://creativecommons.org/about/pdm 54 | subjects: 55 | - !lcc PR 56 | - !lcsh Fantasy 57 | - !lcc PZ 58 | title: Alice's Adventures in Wonderland 59 | url: http://www.gutenberg.org/ebooks/11""" 60 | 61 | #Test submitting a book to the database via the POST route 62 | os.environ['CI'] = '' 63 | r = anon_client.post("/books/post/", data=yaml, content_type='application/octet-stream', secure=True) 64 | self.assertEqual(r.status_code, 401) # unauthenticated 65 | os.environ['CI'] = 'true' 66 | r = anon_client.post("/books/post/", data=yaml, content_type='application/octet-stream', secure=True) 67 | self.assertEqual(r.status_code, 200) # authenticated 68 | os.environ['CI'] = '' 69 | #Test retrieving a book from the database by title 70 | book = Book.objects.get(title="Alice's Adventures in Wonderland") 71 | self.assertEqual(book.title, "Alice's Adventures in Wonderland") 72 | 73 | #Test retrieving an author from the database by name 74 | author = Author.objects.get(name="Carroll, Lewis") 75 | self.assertEqual(author.name, "Carroll, Lewis") 76 | 77 | #Test external links feature 78 | externalLinks = getExternalLinks(book) 79 | self.assertTrue("Librivox" in externalLinks) 80 | #Standard Ebooks test temporarily skipped because of XML parsing error 81 | #self.assertTrue("Standard Ebooks" in externalLinks) 82 | -------------------------------------------------------------------------------- /gitensite/apps/content/views.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from django.views.generic import TemplateView 5 | from django.views.generic import ListView 6 | from django.views.generic import DetailView 7 | from django.views.generic import View 8 | from el_pagination.views import AjaxListView 9 | 10 | from django.http import HttpResponse 11 | from django.http import JsonResponse 12 | from django.views.decorators.csrf import csrf_exempt 13 | from django.utils.decorators import method_decorator 14 | from django.db.models import F 15 | from django.shortcuts import redirect 16 | from django.shortcuts import get_object_or_404 17 | 18 | from gitensite.apps.bookrepos.models import BookRepo 19 | from gitensite.apps.bookinfo.models import Book 20 | from gitensite.apps.bookinfo.db import addBookFromYaml 21 | from gitensite.apps.bookinfo.external import getExternalLinks 22 | 23 | import os 24 | 25 | class HomePageView(TemplateView): 26 | template_name = 'home.html' 27 | 28 | def get_context_data(self, **kwargs): 29 | context = super(HomePageView, self).get_context_data(**kwargs) 30 | 31 | popular = Book.objects.filter(num_downloads__gt=0).order_by("-num_downloads") 32 | context["popular"] = popular[:6] 33 | 34 | return context 35 | 36 | class NewsletterView(TemplateView): 37 | 38 | def get_template_names(self, **kwargs): 39 | return ['newsletters/{issue}.html'.format( 40 | issue=str(self.kwargs['issue']) 41 | )] 42 | 43 | class EbookListingView(TemplateView): 44 | model = Book 45 | template_name = 'listing.html' 46 | 47 | def get_context_data(self, **kwargs): 48 | context = super(EbookListingView, self).get_context_data(**kwargs) 49 | 50 | results = Book.objects.filter(book_id=self.kwargs['bookid']) 51 | if len(results) > 0: 52 | matchedBook = results[0] 53 | context['book'] = matchedBook 54 | sameauthor = Book.objects.filter(author=matchedBook.author).order_by("title") 55 | context['sameauthor'] = sameauthor 56 | 57 | return context 58 | 59 | class SearchView(AjaxListView): 60 | model = Book 61 | template_name = 'book_list.html' 62 | page_template = 'book_list_page.html' 63 | 64 | def get_context_data(self, **kwargs): 65 | context = super(SearchView, self).get_context_data(**kwargs) 66 | 67 | return context 68 | 69 | def get_queryset(self): 70 | if 'q' in self.request.GET: 71 | q = self.request.GET['q'] 72 | searchType = self.request.GET.get('search-type', 'title') 73 | 74 | if searchType == "author": 75 | return super(AjaxListView,self).get_queryset().filter(author__name__icontains=q) 76 | elif searchType == "subjects": 77 | return super(AjaxListView,self).get_queryset().filter(subjects__icontains=q) 78 | else: 79 | return super(AjaxListView,self).get_queryset().filter(title__icontains=q) 80 | else: 81 | return super(AjaxListView,self).get_queryset() 82 | 83 | class BookPostView(View): 84 | @method_decorator(csrf_exempt) 85 | def dispatch(self, request, *args, **kwargs): 86 | return super(BookPostView, self).dispatch(request, *args, **kwargs) 87 | 88 | def post(self, request): 89 | if ("CI" in os.environ and os.environ["CI"] == "true") or ("HTTP_X_GITENBERG_SECRET" in request.META and request.META["HTTP_X_GITENBERG_SECRET"] == os.environ["GITENBERG_SECRET"]): 90 | yaml = request.body 91 | addBookFromYaml(yaml) 92 | return HttpResponse("OK") 93 | else: 94 | return HttpResponse("Incorrect key or key not present", status=401) 95 | 96 | class DownloadView(View): 97 | def get(self, request, bookid): 98 | requested_book = get_object_or_404(Book, book_id=bookid) 99 | 100 | requested_book.num_downloads = F("num_downloads") + 1 101 | requested_book.save() 102 | 103 | return redirect(requested_book.downloads_url) 104 | 105 | class BrowseBooksView(TemplateView): 106 | model = Book 107 | template_name = 'browsebooks.html' 108 | 109 | def get_context_data(self, **kwargs): 110 | context = super(BrowseBooksView, self).get_context_data(**kwargs) 111 | 112 | popular = Book.objects.filter(num_downloads__gt=0).order_by("-num_downloads") 113 | context["popular"] = popular[:12] 114 | 115 | recentlyadded = Book.objects.order_by("-added") 116 | context["recentlyadded"] = recentlyadded[:12] 117 | 118 | recentlyupdated = Book.objects.order_by("-updated") 119 | context["recentlyupdated"] = recentlyupdated[:12] 120 | 121 | return context 122 | 123 | class ExternalLinksView(View): 124 | def get(self, request, bookid): 125 | requested_book = get_object_or_404(Book, book_id=bookid) 126 | 127 | links = getExternalLinks(requested_book) 128 | 129 | return JsonResponse(links) 130 | -------------------------------------------------------------------------------- /gitensite/settings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Django settings for gitensite project. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.7/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.7/ref/settings/ 11 | """ 12 | import os 13 | 14 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 15 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 16 | 17 | # use environment variable to set DJANGO_SECRET_KEY 18 | SECRET_KEY = os.environ['DJANGO_SECRET_KEY'] 19 | 20 | # SECURITY WARNING: don't run with debug turned on in production! 21 | DEBUG = bool(os.environ.get('DJANGO_DEBUG',False)) 22 | 23 | 24 | # Application definition 25 | COMMON_APPS = [ 26 | 'django.contrib.admin', 27 | 'django.contrib.auth', 28 | 'django.contrib.contenttypes', 29 | 'django.contrib.sessions', 30 | 'django.contrib.messages', 31 | 'django.contrib.staticfiles', 32 | 33 | 'django_extensions', 34 | 'foundation', 35 | 'fontawesome', 36 | 'storages', 37 | 'sorl.thumbnail', 38 | 'el_pagination', 39 | 'sh', 40 | 'gitenberg', 41 | ] 42 | 43 | LOCAL_APPS = [ 44 | 'gitensite.apps.content', 45 | 'gitensite.apps.bookrepos', 46 | 'gitensite.apps.bookinfo', 47 | ] 48 | 49 | INSTALLED_APPS = COMMON_APPS + LOCAL_APPS 50 | 51 | MIDDLEWARE = ( 52 | 'debug_toolbar.middleware.DebugToolbarMiddleware', 53 | 'django.contrib.sessions.middleware.SessionMiddleware', 54 | 'django.middleware.common.CommonMiddleware', 55 | 'django.middleware.csrf.CsrfViewMiddleware', 56 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 57 | 'django.contrib.messages.middleware.MessageMiddleware', 58 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 59 | ) 60 | 61 | TEMPLATES = [ 62 | { 63 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 64 | 'APP_DIRS': True, 65 | 'OPTIONS': { 66 | 'context_processors': [ 67 | 'django.contrib.auth.context_processors.auth', 68 | 'django.template.context_processors.debug', 69 | 'django.template.context_processors.i18n', 70 | 'django.template.context_processors.media', 71 | 'django.template.context_processors.static', 72 | 'django.template.context_processors.tz', 73 | 'django.contrib.messages.context_processors.messages', 74 | 'django.template.context_processors.request', 75 | ], 76 | }, 77 | }, 78 | ] 79 | ROOT_URLCONF = 'gitensite.urls' 80 | 81 | WSGI_APPLICATION = 'gitensite.wsgi.application' 82 | 83 | # Database 84 | # https://docs.djangoproject.com/en/1.7/ref/settings/#databases 85 | if 'RDS_HOSTNAME' in os.environ: 86 | DATABASES = { 87 | 'default': { 88 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 89 | 'NAME': os.environ['RDS_DB_NAME'], 90 | 'USER': os.environ['RDS_USERNAME'], 91 | 'PASSWORD': os.environ['RDS_PASSWORD'], 92 | 'HOST': os.environ['RDS_HOSTNAME'], 93 | 'PORT': os.environ['RDS_PORT'], 94 | } 95 | } 96 | else: 97 | DATABASES = { 98 | 'default': { 99 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 100 | 'NAME': 'gitensite', 101 | 'USER': 'postgres', 102 | 'PASSWORD': 'gitensite', 103 | 'HOST': 'localhost', 104 | 'PORT': '5432', 105 | } 106 | } 107 | 108 | # Internationalization 109 | # https://docs.djangoproject.com/en/1.8/topics/i18n/ 110 | LANGUAGE_CODE = 'en-us' 111 | TIME_ZONE = 'UTC' 112 | USE_I18N = True 113 | USE_L10N = True 114 | USE_TZ = False 115 | 116 | 117 | AWS_S3_OBJECT_PARAMETERS = { # see http://developer.yahoo.com/performance/rules.html#expires 118 | 'Expires': 'Thu, 31 Dec 2099 20:00:00 GMT', 119 | 'CacheControl': 'max-age=94608000', 120 | } 121 | AWS_STORAGE_BUCKET_NAME = 'gitensite' 122 | AWS_ACCESS_KEY_ID = 'AKIAIDP7I26XHV4SCSLA' 123 | # use environment variable to set AWS_SECRET_ACCESS_KEY 124 | AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] 125 | 126 | # Tell django-storages that when coming up with the URL for an item in S3 storage, keep 127 | # it simple - just use this domain plus the path. (If this isn't set, things get complicated). 128 | # This controls how the `static` template tag from `staticfiles` gets expanded, if you're using it. 129 | # We also use it in the next setting. 130 | AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME 131 | 132 | # inherit the bucket's ACL 133 | AWS_DEFAULT_ACL = None 134 | 135 | # This is used by the `static` template tag from `static`, if you're using that. Or if anything else 136 | # refers directly to STATIC_URL. So it's safest to always set it. 137 | STATIC_URL = "https://%s/" % AWS_S3_CUSTOM_DOMAIN 138 | 139 | # Tell the staticfiles app to use S3Boto storage when writing the collected static files (when 140 | # you run `collectstatic`). 141 | STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' 142 | DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' 143 | 144 | 145 | # Static files (CSS, JavaScript, Images) 146 | # https://docs.djangoproject.com/en/1.8/howto/static-files/ 147 | STATIC_URL = '/static/' 148 | STATIC_ROOT = os.path.join(BASE_DIR, "static") 149 | STATICFILES_DIRS = ( 150 | os.path.join(BASE_DIR, "assets"), 151 | os.path.join(BASE_DIR, "upload"), 152 | ) 153 | 154 | 155 | # The in-development settings and the default configuration. 156 | if 'ENVIRONMENT' in os.environ and os.environ['ENVIRONMENT'] == 'DEVELOPMENT': 157 | DEBUG = True 158 | 159 | ALLOWED_HOSTS = [] 160 | 161 | INSTALLED_APPS = COMMON_APPS + LOCAL_APPS + [ 162 | 'debug_toolbar', 163 | ] 164 | else: 165 | # django-secure 166 | ALLOWED_HOSTS = ['*'] 167 | USE_X_FORWARDED_HOST = True 168 | 169 | SESSION_COOKIE_SECURE = True 170 | SECURE_SSL_REDIRECT = True 171 | SECURE_HSTS_SECONDS = 3600 172 | # SECURE_HSTS_INCLUDE_SUBDOMAINS = False 173 | # SECURE_FRAME_DENY = True 174 | # SECURE_CONTENT_TYPE_NOSNIFF = False 175 | # SECURE_BROWSER_XSS_FILTER = True 176 | SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') 177 | 178 | 179 | LOGGING = { 180 | 'version': 1, 181 | 'disable_existing_loggers': False, 182 | 'formatters': { 183 | 'verbose': { 184 | 'format': '%(levelname)s %(asctime)s %(module)s %(message)s', 185 | } 186 | }, 187 | 'handlers': { 188 | 'file': { 189 | 'level': 'DEBUG', 190 | 'class': 'logging.handlers.RotatingFileHandler', 191 | # TODO: create and use a /var/log location 192 | 'filename': os.environ['DJANGO_LOG'] if 'DJANGO_LOG' in os.environ else '/var/log/django/django.log', 193 | 'maxBytes': 1024*1024*10, # 10MB 194 | 'backupCount': 5, 195 | }, 196 | 'console': { 197 | 'level': 'DEBUG', 198 | 'class': 'logging.StreamHandler', 199 | 'formatter': 'verbose', 200 | }, 201 | 'mail_admins': { 202 | 'level': 'ERROR', 203 | 'class': 'django.utils.log.AdminEmailHandler', 204 | 'formatter': 'verbose', 205 | } 206 | }, 207 | 'loggers': { 208 | 'django': { 209 | 'handlers': ['file', 'console'], 210 | 'level': 'INFO', 211 | 'propagate': True, 212 | }, 213 | 'django.request': { 214 | 'handlers': ['file', 'console'], 215 | 'level': 'DEBUG', 216 | 'propagate': True, 217 | }, 218 | 'gitensite': { 219 | 'handlers': ['file', 'console'], 220 | 'level': 'DEBUG', 221 | } 222 | }, 223 | } 224 | #MEDIA_URL = '/static/media/' 225 | 226 | MEDIA_ROOT = os.path.join(BASE_DIR, "upload/media") 227 | 228 | import logging 229 | from sorl.thumbnail.log import ThumbnailLogHandler 230 | 231 | handler = ThumbnailLogHandler() 232 | handler.setLevel(logging.ERROR) 233 | logging.getLogger('sorl.thumbnail').addHandler(handler) 234 | 235 | -------------------------------------------------------------------------------- /gitensite/urls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | 5 | from django.conf.urls import include, url 6 | from django.contrib import admin 7 | from django.contrib.auth.decorators import login_required 8 | from django.contrib.staticfiles import views 9 | from django.conf import settings 10 | from django.http import HttpResponse 11 | from django.views.generic.base import TemplateView 12 | 13 | from gitensite.apps.content.views import HomePageView 14 | from gitensite.apps.content.views import NewsletterView 15 | from gitensite.apps.content.views import EbookListingView 16 | from gitensite.apps.content.views import SearchView 17 | from gitensite.apps.content.views import BookPostView 18 | from gitensite.apps.content.views import DownloadView 19 | from gitensite.apps.content.views import BrowseBooksView 20 | from gitensite.apps.content.views import ExternalLinksView 21 | from gitensite.apps.bookinfo.views import all_repos_txt 22 | from gitensite.apps.bookinfo.views import metadata 23 | 24 | @login_required 25 | def msg(request): 26 | message = 'message sent' 27 | return HttpResponse("message sent:" + message) 28 | 29 | urlpatterns = [ 30 | url(r'^admin/', admin.site.urls), 31 | url(r'^newsletter/(?P\d)$', NewsletterView.as_view(), name='newsletter'), 32 | url(r'^book/(?P\d+)$', EbookListingView.as_view(), name='book'), 33 | url(r'^updates/?$', TemplateView.as_view(template_name='updates.html'), name='updates'), 34 | url(r'^books/?$', SearchView.as_view(), name='books'), 35 | url(r'^search/?$', SearchView.as_view(), name='search'), 36 | url(r'^get-involved/?$', TemplateView.as_view(template_name='get-involved.html'), name='get-involved'), 37 | url(r'^faq/?$', TemplateView.as_view(template_name='faq.html'), name='faq'), 38 | url(r'^license/?$', TemplateView.as_view(template_name="license.html"), name='license'), 39 | url(r'^all_repos.txt$', all_repos_txt, name='all_repos.txt'), 40 | url(r'^$', HomePageView.as_view(), name='home'), 41 | url(r'^books/(?P\d+)\.(?Pjson|yaml)$', metadata, name='metadata'), 42 | url(r'^books/post/', BookPostView.as_view(), name='book-post'), 43 | url(r'^download/(?P\d+)$', DownloadView.as_view(), name='download'), 44 | url(r'^browse/?$', BrowseBooksView.as_view(), name='browse'), 45 | url(r'^external/(?P\d+)$', ExternalLinksView.as_view(), name='external'), 46 | url(r"^msg/$", msg, name="msg"), 47 | ] 48 | 49 | if settings.DEBUG: 50 | urlpatterns += [ 51 | url(r'^static/(?P.*)$', views.serve), 52 | ] 53 | 54 | -------------------------------------------------------------------------------- /gitensite/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for gitensite project. 3 | It exposes the WSGI callable as a module-level variable named ``application``. 4 | For more information on this file, see 5 | 6 | """ 7 | import os 8 | 9 | 10 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gitensite.settings') 11 | os.environ.setdefault('ENVIRONMENT', 'NOT_PRODUCTION') 12 | os.environ.setdefault('DJANGO_LOG', '/var/log/django/django.log') 13 | 14 | from django.core.wsgi import get_wsgi_application 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | 6 | if __name__ == "__main__": 7 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gitensite.settings') 8 | 9 | from django.core.management import execute_from_command_line 10 | 11 | execute_from_command_line(sys.argv) 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.3 2 | beautifulsoup4==4.11.1 3 | boto3==1.21.40 4 | botocore==1.24.40 5 | CacheControl==0.12.10 6 | cairocffi==0.8.0 7 | certifi==2021.10.8 8 | cffi==1.15.0 9 | charset-normalizer==2.0.12 10 | cryptography==36.0.2 11 | distlib==0.3.4 12 | dj-database-url==0.5.0 13 | Django==2.2.15 14 | django-debug-toolbar==3.2.4 15 | django-el-pagination==3.3.0 16 | django-extensions==3.1.5 17 | django-fontawesome==1.0 18 | django-storages==1.12.3 19 | django-zurb-foundation==5.5.0 20 | docopt==0.6.2 21 | filelock==3.6.0 22 | gitberg==0.8.0 23 | gitdb==4.0.9 24 | github3.py==3.2.0 25 | GitPython==3.1.27 26 | idna==3.3 27 | isodate==0.6.1 28 | Jinja2==3.1.1 29 | jmespath==1.0.0 30 | lxml==4.8.0 31 | MarkupSafe==2.1.1 32 | mock==2.0.0 33 | msgpack==1.0.3 34 | packaging==21.3 35 | pbr==5.8.1 36 | Pillow==9.1.0 37 | platformdirs==2.5.1 38 | pluggy==1.0.0 39 | psycopg2-binary>=2.8,<2.9 40 | py==1.11.0 41 | pycparser==2.21 42 | PyJWT==2.3.0 43 | pymarc==4.2.0 44 | pyOpenSSL==22.0.0 45 | pyparsing==3.0.8 46 | python-dateutil==2.8.2 47 | pytz==2022.1 48 | PyYAML==6.0 49 | rdflib==6.1.1 50 | requests==2.27.1 51 | s3transfer==0.5.2 52 | semver==2.2.0 53 | sh==1.14.2 54 | six==1.16.0 55 | smmap==5.0.0 56 | sorl-thumbnail==12.8.0 57 | soupsieve==2.3.2.post1 58 | SPARQLWrapper==2.0.0 59 | sqlparse==0.4.2 60 | toml==0.10.2 61 | tox==3.25.0 62 | uritemplate==4.1.1 63 | urllib3==1.26.9 64 | virtualenv==20.14.1 65 | wikipedia==1.4.0 -------------------------------------------------------------------------------- /static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/static/.gitkeep -------------------------------------------------------------------------------- /upload/null: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitenberg-dev/giten_site/ccd18a16becb3e6a97ca9235f25982db0e1bc1cf/upload/null --------------------------------------------------------------------------------