├── screenshots ├── home.png ├── dataset.png ├── gruppi.png └── organizzazioni.png ├── default_locale ├── docker-compose.harvest.yml ├── data ├── init │ └── groups │ │ ├── salute.json │ │ ├── ambiente.json │ │ ├── energia.json │ │ ├── trasporti.json │ │ ├── economia.json │ │ ├── regioni.json │ │ ├── scienza.json │ │ ├── societa.json │ │ ├── cultura.json │ │ ├── governo.json │ │ ├── internazionali.json │ │ ├── giustizia.json │ │ └── agricoltura.json └── config │ ├── theme_to_group.ini │ └── topics.json ├── docker-compose.build.yml ├── ckan-harvest.sh ├── harvest_fetch_and_gather.sh ├── CHANGELOG.md ├── LICENSE ├── .gitmodules ├── AUTHORS ├── docker-compose.yml ├── publiccode.yml ├── ckan-init.sh ├── Dockerfile ├── ckan-entrypoint.sh ├── ckan.ini └── README.md /screenshots/home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/italia/ckan-it/HEAD/screenshots/home.png -------------------------------------------------------------------------------- /screenshots/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/italia/ckan-it/HEAD/screenshots/dataset.png -------------------------------------------------------------------------------- /screenshots/gruppi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/italia/ckan-it/HEAD/screenshots/gruppi.png -------------------------------------------------------------------------------- /default_locale: -------------------------------------------------------------------------------- 1 | LC_ALL=en_US.UTF-8 2 | LANG=en_US.UTF-8 3 | LC_CTYPE=en_US.UTF-8 4 | LC_COLLATE=en_US.UTF-8 5 | -------------------------------------------------------------------------------- /screenshots/organizzazioni.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/italia/ckan-it/HEAD/screenshots/organizzazioni.png -------------------------------------------------------------------------------- /docker-compose.harvest.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | ckan: 6 | environment: 7 | CKAN_HARVEST: "true" 8 | -------------------------------------------------------------------------------- /data/init/groups/salute.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "salute", 3 | "description" : "Salute", 4 | "title" : "Salute" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/ambiente.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "ambiente", 3 | "description" : "Ambiente", 4 | "title" : "Ambiente" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/energia.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "energia", 3 | "description" : "Energia", 4 | "title" : "Energia" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/trasporti.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "trasporti", 3 | "description" : "Trasporti", 4 | "title" : "Trasporti" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/economia.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "economia", 3 | "description" : "Economia e finanze", 4 | "title" : "Economia" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/regioni.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "regioni", 3 | "description" : "Regioni e città", 4 | "title" : "Regioni" 5 | } 6 | 7 | -------------------------------------------------------------------------------- /data/init/groups/scienza.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "scienza", 3 | "description" : "Scienza e tecnologia", 4 | "title" : "Scienza" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/societa.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "societa", 3 | "description" : "Popolazione e società", 4 | "title" : "Società" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/cultura.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "cultura", 3 | "description" : "Istruzione, cultura e sport", 4 | "title" : "Cultura" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/governo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "governo", 3 | "description" : "Governo e settore pubblico", 4 | "title" : "Governo" 5 | } 6 | 7 | 8 | -------------------------------------------------------------------------------- /data/init/groups/internazionali.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "internazionali", 3 | "description" : "Tematiche internazionali", 4 | "title" : "Internazionali" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/giustizia.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "giustizia", 3 | "description" : "Giustizia, sistema giuridico e sicurezza pubblica", 4 | "title" : "Giustizia" 5 | } 6 | -------------------------------------------------------------------------------- /data/init/groups/agricoltura.json: -------------------------------------------------------------------------------- 1 | { 2 | "title" : "Agricoltura", 3 | "description" : "Agricoltura, pesca, silvicoltura e prodotti alimentari", 4 | "name" : "agricoltura" 5 | } 6 | -------------------------------------------------------------------------------- /docker-compose.build.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | ckan: 6 | build: 7 | context: ./ 8 | image: ckan-it:2.6.8-dev.2 9 | 10 | solr: 11 | build: 12 | context: ./solr 13 | image: ckan-it-solr:2.3.0-dev.1 14 | -------------------------------------------------------------------------------- /ckan-harvest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CONFIG="${CKAN_CONFIG}/ckan.ini" 4 | 5 | if [ ! -z "$1" ]; then 6 | paster --plugin=ckanext-harvest harvester $1 --config=$CONFIG 7 | else 8 | paster --plugin=ckanext-harvest harvester run --config=$CONFIG 9 | paster --plugin=ckanext-harvest harvester job-all --config=$CONFIG 10 | fi 11 | -------------------------------------------------------------------------------- /data/config/theme_to_group.ini: -------------------------------------------------------------------------------- 1 | # Bind a theme to one ore more groups: 2 | # THEME_NAME = groupname [, groupname ...] 3 | 4 | [dcatapit:theme_group_mapping] 5 | 6 | AGRI = agricoltura 7 | EDUC = cultura 8 | ENVI = ambiente 9 | ENER = energia 10 | TRAN = trasporti 11 | TECH = scienza 12 | ECON = economia 13 | SOCI = societa 14 | HEAL = salute 15 | GOVE = governo 16 | REGI = regioni 17 | JUST = giustizia 18 | INTR = internazionali 19 | 20 | -------------------------------------------------------------------------------- /harvest_fetch_and_gather.sh: -------------------------------------------------------------------------------- 1 | #!/bin/env bash 2 | 3 | # Launch the harvesting processes 4 | # Name of the process as first parameter (gather_consumer, fetch_consumer) 5 | 6 | config="${CKAN_CONFIG}/ckan.ini" 7 | 8 | if [ -z ${1+x} ] 9 | then 10 | echo "Process name parameter required" 11 | exit 1 12 | fi 13 | 14 | process="$1" 15 | 16 | while true 17 | do 18 | paster --plugin=ckanext-harvest harvester "$process" -c "${config}" 19 | sleep 5 20 | done 21 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | Version number follows the semantic versioning specifications: 4 | X.Y.Z-P where X.Y.Z is equal to CKAN official release and P is our custom distribution version. 5 | 6 | ## 2.6.8-2 2019-10-14 7 | 8 | * downgrade ckanext-harvest from v1.1.2 to v1.1.1 (fix issues in rendering /harvest pages) 9 | 10 | ## 2.6.8-1 2019-10-14 11 | 12 | * upgrade CKAN from v2.6.7 to v2.6.8 13 | * downgrade ckanext-harvest from v1.1.4 to v1.1.2 (fix issues in catalogues list) 14 | * update ckanext-spatial from 8a4d63fdb28a1689f14f9e228b8b41ca226866cf commit to 2.6.8-1 tag (fix issue in tag cleaning during harvesting) 15 | 16 | ## 2.6.7-1 2019-10-01 17 | 18 | First distribution release after heavy refactoring of old codebase. Based on CKAN v2.6.7. 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | CKAN for Italian Open Data 2 | Copyright (c) 2019 Presidenza del Consiglio dei Ministri 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published 6 | by the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU Affero General Public License for more details. 13 | 14 | You should have received a copy of the GNU Affero General Public License 15 | along with this program. If not, see . 16 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ckanext-dcat"] 2 | path = ckanext-dcat 3 | url = https://github.com/italia/ckanext-dcat.git 4 | [submodule "ckanext-dcatapit"] 5 | path = ckanext-dcatapit 6 | url = https://github.com/italia/ckanext-dcatapit.git 7 | [submodule "ckanext-harvest"] 8 | path = ckanext-harvest 9 | url = https://github.com/italia/ckanext-harvest.git 10 | [submodule "ckanext-spatial"] 11 | path = ckanext-spatial 12 | url = https://github.com/italia/ckanext-spatial.git 13 | [submodule "ckanext-multilang"] 14 | path = ckanext-multilang 15 | url = https://github.com/italia/ckanext-multilang.git 16 | [submodule "datapusher"] 17 | path = datapusher 18 | url = https://github.com/italia/datapusher.git 19 | [submodule "ckan"] 20 | path = ckan 21 | url = https://github.com/italia/ckan.git 22 | [submodule "data/init/harvesters"] 23 | path = data/init/harvesters 24 | url = https://github.com/italia/ckan-it-harvesters.git 25 | [submodule "solr"] 26 | path = solr 27 | url = https://github.com/italia/ckan-it-solr.git 28 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Presidenza del Consiglio dei Ministri 2 | 3 | Owner 4 | ----- 5 | Team per la Trasformazione Digitale 6 | Github: https://github.com/italia 7 | Forum: https://forum.italia.it/c/daf 8 | Slack: #daf on https://developersitalia.slack.com 9 | Twitter: https://twitter.com/teamdigitaleIT 10 | 11 | Maintainer 12 | ----------- 13 | Alessio Cimarelli (alessio.cimarelli@gmail.com) 14 | 15 | Contributors 16 | ------------ 17 | Giovanni Bajo (rasky@develer.com) 18 | Francesco Piero Paolicelli (piersoft@me.com) 19 | Gianluca Varisco (gianluca.varisco@gmail.com) 20 | Alessandro Ercolani (ale.ercolani@gmail.com) 21 | Riccardo Iaconelli (riccardo@kde.org) 22 | Tobia Di Pisa (tobia.dipisa@geo-solutions.it) 23 | 24 | Original Authors 25 | ---------------- 26 | GeoSolutions SAS (https://www.geo-solutions.it/) 27 | David Greco (greco@acm.org) 28 | Alessandro Parma () 29 | Emanuele Tajariol (emanuele.tajariol@geo-solutions.it) 30 | 31 | The version control system provides attribution for specific lines of code. 32 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | ckan: 6 | image: italia/ckan-it:2.6.8-2 7 | container_name: italia-ckan-it 8 | environment: 9 | CKAN_DEBUG: "false" 10 | CKAN_SITE_URL: http://localhost:5000 11 | CKAN_ADMIN_EMAIL: ckanadmin@mail.com 12 | CKAN_ADMIN_USERNAME: ckanadmin 13 | CKAN_ADMIN_PASSWORD: ckanpassword 14 | CKAN_DB_HOST: db 15 | CKAN_DB_PORT: 5432 16 | CKAN_DB_USER: ckan 17 | PGPASSWORD: ckan 18 | CKAN_SQLALCHEMY_URL: postgresql://ckan:ckan@db:5432/ 19 | CKAN_REDIS_HOST: redis 20 | CKAN_REDIS_PORT: 6379 21 | CKAN_REDIS_URL: redis://redis:6379/ 22 | CKAN_SOLR_HOST: solr 23 | CKAN_SOLR_PORT: 8983 24 | CKAN_SOLR_URL: http://solr:8983/solr/ckan 25 | stdin_open: true 26 | tty: true 27 | ports: 28 | - 5000:5000/tcp 29 | depends_on: 30 | - db 31 | - redis 32 | - solr 33 | volumes: 34 | - ckanlog:/var/log/ckan 35 | 36 | solr: 37 | image: italia/ckan-it-solr:2.3.0-1 38 | container_name: italia-ckan-it-solr 39 | stdin_open: true 40 | tty: true 41 | volumes: 42 | - solrdb:/opt/solr/server/solr/ckan/data 43 | 44 | redis: 45 | image: redis:5.0.5 46 | container_name: italia-ckan-it-redis 47 | stdin_open: true 48 | tty: true 49 | 50 | db: 51 | image: ckan/postgresql:latest 52 | container_name: italia-ckan-it-db 53 | environment: 54 | POSTGRES_USER: ckan 55 | POSTGRES_PASSWORD: ckan 56 | POSTGRES_DB: ckan 57 | PGDATA: /var/lib/postgresql/data/pgdata 58 | ports: 59 | - 5432:5432/tcp 60 | stdin_open: true 61 | tty: true 62 | volumes: 63 | - pgsqldb:/var/lib/postgresql/data 64 | 65 | volumes: 66 | pgsqldb: 67 | solrdb: 68 | ckanlog: 69 | -------------------------------------------------------------------------------- /publiccode.yml: -------------------------------------------------------------------------------- 1 | # This repository adheres to the publiccode.yml standard by including this 2 | # metadata file that makes public software easily discoverable. 3 | # More info at https://github.com/italia/publiccode.yml 4 | 5 | publiccodeYmlVersion: '0.2' 6 | name: 'CKAN for Italian Open Data' 7 | releaseDate: '2019-10-14' 8 | url: 'https://github.com/italia/ckan-it.git' 9 | isBasedOn: 'https://github.com/ckan/ckan.git' 10 | softwareVersion: dev 11 | developmentStatus: development 12 | softwareType: standalone/web 13 | platforms: 14 | - web 15 | categories: 16 | - data-analytics 17 | - data-collection 18 | - data-visualization 19 | legal: 20 | license: 'AGPL-3.0-or-later' 21 | repoOwner: 'Team per la Trasformazione Digitale' 22 | authorsFile: 'AUTHORS' 23 | maintenance: 24 | type: 'internal' 25 | contacts: 26 | - name: 'Alessio Cimarelli' 27 | email: 'alessio.cimarelli@gmail.com' 28 | 29 | intendedAudience: 30 | scope: 31 | - science-and-technology 32 | - research 33 | - welfare 34 | - government 35 | countries: 36 | - it 37 | localisation: 38 | localisationReady: false 39 | availableLanguages: 40 | - it 41 | it: 42 | countryExtensionVersion: '0.2' 43 | riuso: 44 | codiceIPA: PCM 45 | description: 46 | en: 47 | genericName: 'Data Management System' 48 | shortDescription: >- 49 | This project provides everything you need to run CKAN plus a set of 50 | extensions for supporting Italian open data in a set of Docker images. 51 | longDescription: > 52 | This project provides everything you need to run CKAN plus a set of 53 | extensions for supporting Italian open data. You can run the full stack 54 | using docker-compose to orchestrate four containers providing CKAN 2.6.8, Solr 6.2, 55 | Redis 5.0.5, PostgreSQL 10.1. 56 | 57 | "Warning: this product is under active development and it is to be considered unstable." 58 | It has been currently used only in testing environments, 59 | but we think it can provide a good base for running a production service 60 | (please feel free to contribute with pull requests to this end). 61 | features: 62 | - Statistics 63 | - Text view 64 | - Image view 65 | - Harvest 66 | screenshots: 67 | - screenshots/home.png 68 | - screenshots/dataset.png 69 | - screenshots/organizzazioni.png 70 | - screenshots/gruppi.png 71 | -------------------------------------------------------------------------------- /ckan-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/env bash 2 | 3 | # Wait until CKAN REST APIs are ready, then create groups from data/init/groups 4 | add_groups () { 5 | until $(curl --output /dev/null --silent --head --fail "${CKAN_SITE_URL}"); do 6 | echo "CKAN is not ready, yet. Trying again in two seconds." 7 | sleep 2 8 | done 9 | 10 | apikey=$(psql -q -t -h ${CKAN_DB_HOST} -U ckan -d ${CKAN_DB_USER} -c "select apikey from public.user where name='${CKAN_ADMIN_USERNAME}';") 11 | 12 | for file in "${CKAN_HOME}"/data/init/groups/*.json; do 13 | echo "Creating group from file ${file}" 14 | curl -i -H "X-CKAN-API-Key: ${apikey}" -XPOST -d @$file "${CKAN_SITE_URL}"/api/3/action/group_create 15 | done 16 | } 17 | 18 | # Wait until CKAN REST APIs are ready, then create orgs from data/init/harvesters/orgs 19 | add_orgs () { 20 | until $(curl --output /dev/null --silent --head --fail "${CKAN_SITE_URL}"); do 21 | echo "CKAN is not ready, yet. Trying again in two seconds." 22 | sleep 2 23 | done 24 | 25 | apikey=$(psql -q -t -h ${CKAN_DB_HOST} -U ckan -d ${CKAN_DB_USER} -c "select apikey from public.user where name='${CKAN_ADMIN_USERNAME}';") 26 | 27 | for file in "${CKAN_HOME}"/data/init/harvesters/orgs/*.json; do 28 | echo "Creating organization from file ${file}" 29 | curl -i -H "X-CKAN-API-Key: ${apikey}" -XPOST -d @$file "${CKAN_SITE_URL}"/api/3/action/organization_create 30 | done 31 | } 32 | 33 | # Wait until CKAN REST APIs are ready, then create sources from data/init/harvesters/sources 34 | add_sources () { 35 | until $(curl --output /dev/null --silent --head --fail "${CKAN_SITE_URL}"); do 36 | echo "CKAN is not ready, yet. Trying again in two seconds." 37 | sleep 2 38 | done 39 | 40 | apikey=$(psql -q -t -h ${CKAN_DB_HOST} -U ckan -d ${CKAN_DB_USER} -c "select apikey from public.user where name='${CKAN_ADMIN_USERNAME}';") 41 | 42 | for file in "${CKAN_HOME}"/data/init/harvesters/sources/*.json; do 43 | echo "Creating source from file ${file}" 44 | curl -i -H "X-CKAN-API-Key: ${apikey}" -XPOST -d @$file "${CKAN_SITE_URL}"/api/3/action/package_create 45 | done 46 | } 47 | 48 | path_to_eurovoc="${CKAN_HOME}/src/ckanext-dcatapit/examples/eurovoc.rdf" 49 | config="${CKAN_CONFIG}/ckan.ini" 50 | 51 | # Add a local admin user and add it to the sysadmin group. 52 | paster --plugin=ckan user add "${CKAN_ADMIN_USERNAME}" email="${CKAN_ADMIN_EMAIL}" password="${CKAN_ADMIN_PASSWORD}" --config "${config}" 53 | paster --plugin=ckan sysadmin add "${CKAN_ADMIN_USERNAME}" --config "${config}" 54 | 55 | # Load Vocabulary 56 | paster --plugin=ckanext-dcatapit vocabulary load --url http://publications.europa.eu/resource/distribution/language/rdf/skos_core/languages-skos.rdf --name languages --config "$config" 57 | paster --plugin=ckanext-dcatapit vocabulary load --url http://publications.europa.eu/resource/distribution/data-theme/rdf/skos_core/data-theme-skos.rdf --name eu_themes --config "$config" 58 | paster --plugin=ckanext-dcatapit vocabulary load --url http://publications.europa.eu/resource/distribution/place/rdf/skos_core/places-skos.rdf --name places --config "$config" 59 | paster --plugin=ckanext-dcatapit vocabulary load --url http://publications.europa.eu/resource/distribution/frequency/rdf/skos_core/frequencies-skos.rdf --name frequencies --config "$config" 60 | paster --plugin=ckanext-dcatapit vocabulary load --url http://publications.europa.eu/resource/distribution/file-type/rdf/skos_core/filetypes-skos.rdf --name filetype --config "$config" 61 | 62 | wget "https://raw.githubusercontent.com/italia/daf-ontologie-vocabolari-controllati/master/VocabolariControllati/territorial-classifications/regions/regions.rdf" -O "/tmp/regions.rdf" 63 | paster --plugin=ckanext-dcatapit vocabulary load --filename "/tmp/regions.rdf" --name regions --config "${config}" 64 | 65 | wget "https://raw.githubusercontent.com/italia/daf-ontologie-vocabolari-controllati/master/VocabolariControllati/licences/licences.rdf" -O "/tmp/licences.rdf" 66 | paster --plugin=ckanext-dcatapit vocabulary load --filename "/tmp/licences.rdf" --name licenses --config "${config}" 67 | 68 | wget "https://raw.githubusercontent.com/italia/daf-ontologie-vocabolari-controllati/master/VocabolariControllati/theme-subtheme-mapping/theme-subtheme-mapping.rdf" -O "/tmp/theme-subtheme-mapping.rdf" 69 | paster --plugin=ckanext-dcatapit vocabulary load --filename "/tmp/theme-subtheme-mapping.rdf" --name subthemes --config "${config}" "$path_to_eurovoc" 70 | 71 | add_groups 72 | if [ "${CKAN_HARVEST}" = "true" -a -d "${CKAN_HOME}/data/init/harvesters/" ]; then 73 | add_orgs 74 | add_sources 75 | fi 76 | 77 | echo -e "\nCKAN init completed successfully" 78 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | 3 | # Set vars 4 | ENV CKAN_HOME /usr/lib/ckan/default 5 | ENV CKAN_CONFIG /etc/ckan/default 6 | ENV CKAN_LOG_DIR /var/log/ckan 7 | ENV CKAN_STORAGE_PATH /var/lib/ckan 8 | ENV CKAN_SITE_URL http://localhost:5000 9 | 10 | # Install requirements 11 | RUN yum -y install epel-release 12 | RUN yum -y install postgresql postgresql-contrib postgresql-devel postgis 13 | RUN yum -y install gcc gcc-c++ make git gdal geos 14 | RUN yum -y install libxml2 libxml2-devel libxslt libxslt-devel 15 | RUN yum -y install gdal-python python-pip python-imaging python-virtualenv \ 16 | libxml2-python libxslt-python python-lxml \ 17 | python-devel python-babel \ 18 | python-pylons python-repoze-who python-repoze-who-plugins-sa \ 19 | python-repoze-who-testutil python-repoze-who-friendlyform \ 20 | python-tempita python-zope-interface policycoreutils-python 21 | RUN yum -y install gettext 22 | RUN yum -y install wget 23 | 24 | RUN yum -y groupinstall "Development Tools" 25 | 26 | RUN wget http://download.redis.io/redis-stable.tar.gz \ 27 | && tar xvzf redis-stable.tar.gz \ 28 | && pushd redis-stable \ 29 | && make \ 30 | && cp src/redis-cli /usr/local/bin/ \ 31 | && chmod 755 /usr/local/bin/redis-cli \ 32 | && popd \ 33 | && rm -rf ./redis-stable* 34 | 35 | # Upgrade python pip 36 | RUN pip install --upgrade pip 37 | 38 | # Setup ckan Directory 39 | RUN mkdir -p "${CKAN_HOME}" 40 | RUN mkdir -p "${CKAN_LOG_DIR}" 41 | RUN mkdir -p "${CKAN_CONFIG}" 42 | RUN mkdir -p "${CKAN_STORAGE_PATH}" 43 | 44 | # Add CKAN source code 45 | RUN mkdir -p "${CKAN_HOME}/src/ckan/" 46 | ADD ./ckan ${CKAN_HOME}/src/ckan/ 47 | 48 | # Add ckan user 49 | RUN useradd --home "${CKAN_HOME}" --shell /bin/bash ckan 50 | 51 | # Set permissions to CKAN folders 52 | # Fix folders' owners and permissions 53 | RUN chown ckan:ckan "${CKAN_HOME}" -R \ 54 | && chown ckan:ckan "${CKAN_CONFIG}" -R \ 55 | && chown ckan:ckan "${CKAN_LOG_DIR}" -R \ 56 | && chown ckan:ckan "${CKAN_STORAGE_PATH}" -R 57 | 58 | RUN chmod 755 "${CKAN_HOME}" -R \ 59 | && chmod 755 "${CKAN_CONFIG}" -R \ 60 | && chmod 755 "${CKAN_LOG_DIR}" -R \ 61 | && chmod 755 "${CKAN_STORAGE_PATH}" -R 62 | 63 | 64 | # Temporary fix for dependencies 65 | RUN pip install pytz diagnostics 66 | 67 | # Remove old versions of python packages installed through distutils 68 | RUN rm -rf /usr/bin/markdown* 69 | RUN rm -rf /usr/lib/python2.7/site-packages/markdown* 70 | RUN rm -rf /usr/lib/python2.7/site-packages/webhelpers/markdown* 71 | RUN rm -rf /usr/lib/python2.7/site-packages/Markdown-2.4.1-py2.7.egg-info 72 | RUN rm -rf /usr/share/doc/python2-markdown* 73 | RUN rm -rf /usr/share/licenses/python2-markdown* 74 | 75 | # Install python packages requirements 76 | RUN pip install "setuptools==36.1" 77 | RUN pip install -r "${CKAN_HOME}/src/ckan/requirements.txt" 78 | 79 | # Install CKAN 80 | RUN pip install -e "${CKAN_HOME}/src/ckan" #egg=ckan 81 | 82 | # Install ckanext-harvest 83 | RUN mkdir "${CKAN_HOME}/src/ckanext-harvest/" 84 | ADD ./ckanext-harvest/ ${CKAN_HOME}/src/ckanext-harvest/ 85 | RUN pip install -e "${CKAN_HOME}/src/ckanext-harvest/" 86 | RUN pip install -r "${CKAN_HOME}/src/ckanext-harvest/pip-requirements.txt" 87 | RUN pip install -r "${CKAN_HOME}/src/ckanext-harvest/dev-requirements.txt" 88 | 89 | # Install ckanext-dcat 90 | RUN mkdir "${CKAN_HOME}/src/ckanext-dcat/" 91 | ADD ./ckanext-dcat/ ${CKAN_HOME}/src/ckanext-dcat/ 92 | RUN pip install -e "${CKAN_HOME}/src/ckanext-dcat/" 93 | RUN pip install -r "${CKAN_HOME}/src/ckanext-dcat/requirements.txt" 94 | 95 | # Install ckanext-dcatapit 96 | RUN mkdir "${CKAN_HOME}/src/ckanext-dcatapit/" 97 | ADD ./ckanext-dcatapit/ ${CKAN_HOME}/src/ckanext-dcatapit/ 98 | RUN pip install -e "${CKAN_HOME}/src/ckanext-dcatapit/" 99 | RUN pip install -r "${CKAN_HOME}/src/ckanext-dcatapit/dev-requirements.txt" 100 | 101 | # Install ckanext-spatial 102 | RUN mkdir "${CKAN_HOME}/src/ckanext-spatial/" 103 | ADD ./ckanext-spatial/ ${CKAN_HOME}/src/ckanext-spatial/ 104 | RUN pip install -e "${CKAN_HOME}/src/ckanext-spatial/" 105 | RUN pip install -r "${CKAN_HOME}/src/ckanext-spatial/pip-requirements.txt" 106 | 107 | # Install ckanext-multilang 108 | RUN mkdir "${CKAN_HOME}/src/ckanext-multilang/" 109 | ADD ./ckanext-multilang/ ${CKAN_HOME}/src/ckanext-multilang/ 110 | RUN pip install -e "${CKAN_HOME}/src/ckanext-multilang/" 111 | 112 | # DCATAPIT theme to group mapping file 113 | ADD ./data/config/theme_to_group.ini ${CKAN_CONFIG} 114 | RUN chmod 666 "${CKAN_CONFIG}/theme_to_group.ini" 115 | 116 | # CKAN group to DCATAPIT theme mapping file 117 | ADD ./data/config/topics.json ${CKAN_CONFIG} 118 | RUN chmod 666 "${CKAN_CONFIG}/topics.json" 119 | 120 | # Copy various data and initialization files 121 | RUN mkdir "${CKAN_HOME}/data/" 122 | ADD ./data ${CKAN_HOME}/data/ 123 | 124 | # Link to who.ini 125 | RUN ln -s "${CKAN_HOME}/src/ckan/ckan/config/who.ini" "${CKAN_CONFIG}/who.ini" 126 | 127 | # Setup entrypoint 128 | COPY ./ckan-entrypoint.sh / 129 | RUN chmod +x /ckan-entrypoint.sh 130 | ENTRYPOINT ["/ckan-entrypoint.sh"] 131 | 132 | # Add startup scripts 133 | ADD ./ckan-init.sh / 134 | RUN chmod +x /ckan-init.sh 135 | ADD ./harvest_fetch_and_gather.sh / 136 | RUN chmod +x /harvest_fetch_and_gather.sh 137 | ADD ./ckan-harvest.sh / 138 | RUN chmod +x /ckan-harvest.sh 139 | 140 | # Set default user and work directory 141 | # USER ckan 142 | WORKDIR "${CKAN_CONFIG}" 143 | 144 | EXPOSE 5000 145 | -------------------------------------------------------------------------------- /ckan-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/env bash 2 | 3 | set -e 4 | 5 | config="${CKAN_CONFIG}/ckan.ini" 6 | 7 | wait_for_services () { 8 | until psql -h "${CKAN_DB_HOST}" -U "${CKAN_DB_USER}" -c '\q'; do 9 | echo "Postgres is not ready, yet. Trying again in two seconds." 10 | sleep 2 11 | done 12 | 13 | until curl -f "http://${CKAN_SOLR_HOST}:${CKAN_SOLR_PORT}"; do 14 | echo "Solr is not ready, yet. Trying again in two seconds." 15 | sleep 2 16 | done 17 | 18 | until redis-cli -h "${CKAN_REDIS_HOST}" -p "${CKAN_REDIS_PORT}" <<< ping | grep "PONG"; do 19 | echo "Redis is not ready, yet. Trying again in two seconds." 20 | sleep 2 21 | done 22 | 23 | echo "All dependencies are ready." 24 | } 25 | 26 | write_config () { 27 | # Create config 28 | paster make-config --no-interactive ckan ${config} 29 | 30 | # Edit DEFAULT section 31 | paster --plugin=ckan config-tool ${config} -s "DEFAULT" "debug = ${CKAN_DEBUG:=false}" 32 | 33 | # Edit app:main section 34 | paster --plugin=ckan config-tool ${config} -s "app:main" \ 35 | "ckan.harvest.mq.type = redis" \ 36 | "ckan.harvest.mq.hostname = redis" \ 37 | "sqlalchemy.url = ${CKAN_SQLALCHEMY_URL}" \ 38 | "ckan.site_url = ${CKAN_SITE_URL}" \ 39 | "ckan.auth.user_create_organizations = true" \ 40 | "ckanext.dcat.rdf.profiles = euro_dcat_ap it_dcat_ap" \ 41 | "ckanext.dcat.base_uri = ${CKAN_DCAT_BASE_URI}" \ 42 | "ckanext.dcat.expose_subcatalogs = True" \ 43 | "ckanext.dcat.clean_tags = True" \ 44 | "ckanext.dcatapit.theme_group_mapping.file = ${CKAN_CONFIG}/theme_to_group.ini" \ 45 | "ckanext.dcatapit.nonconformant_themes_mapping.file = ${CKAN_CONFIG}/topics.json" \ 46 | "geonames.username = demo" \ 47 | "geonames.limits.countries = IT" \ 48 | "ckan.site_id = dcatapit_docker_default" \ 49 | "solr_url=${CKAN_SOLR_URL}" \ 50 | "ckan.redis.url = ${CKAN_REDIS_URL}" \ 51 | "ckan.cors.origin_allow_all = true" \ 52 | "ckan.plugins = stats text_view image_view recline_view spatial_metadata spatial_query harvest ckan_harvester multilang multilang_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface dcatapit_pkg dcatapit_org dcatapit_config dcatapit_harvester dcatapit_ckan_harvester dcatapit_csw_harvester dcatapit_harvest_list dcatapit_subcatalog_facets dcatapit_theme_group_mapper" \ 53 | "ckan.spatial.srid = 4326" \ 54 | "ckan.locale_default = it" \ 55 | "ckan.locale_order = it de fr en pt_BR ja cs_CZ ca es el sv sr sr@latin no sk fi ru pl nl bg ko_KR hu sa sl lv" \ 56 | "ckan.locales_offered = it de fr en" \ 57 | "ckan.locales_filtered_out = it_IT" 58 | 59 | # Edit handlers section 60 | paster --plugin=ckan config-tool ${config} -s "handlers" \ 61 | "keys = console, file" 62 | 63 | # Edit logger_root section 64 | paster --plugin=ckan config-tool ${config} -s "logger_root" \ 65 | "handlers = console, file" 66 | 67 | # Edit logger_ckan section 68 | paster --plugin=ckan config-tool ${config} -s "logger_ckan" \ 69 | "handlers = console, file" 70 | 71 | # Edit logger_ckanext section 72 | paster --plugin=ckan config-tool ${config} -s "logger_ckanext" \ 73 | "handlers = console, file" 74 | 75 | # Edit handler_file section 76 | paster --plugin=ckan config-tool ${config} -s "handler_file" \ 77 | "class = logging.handlers.RotatingFileHandler" \ 78 | "formatter = generic" \ 79 | "level = NOTSET" \ 80 | "args = (\"${CKAN_LOG_DIR}/ckan.log\", \"a\", 20000000, 9)" 81 | } 82 | 83 | init_db () { 84 | # Initializes the database 85 | paster --plugin=ckan db init -c "${config}" 86 | 87 | # Initialize harvester database 88 | paster --plugin=ckanext-harvest harvester initdb -c "${config}" 89 | 90 | # Inizialize dcat-ap-it database 91 | paster --plugin=ckanext-dcatapit vocabulary initdb -c "${config}" 92 | 93 | # Setup multilang database 94 | paster --plugin=ckanext-multilang multilangdb initdb -c "${config}" 95 | } 96 | 97 | harvesting () { 98 | nohup /harvest_fetch_and_gather.sh gather_consumer &> "${CKAN_LOG_DIR}"/gather_consumer & 99 | nohup /harvest_fetch_and_gather.sh fetch_consumer &> "${CKAN_LOG_DIR}"/fetch_consumer & 100 | } 101 | 102 | ckan_configure () { 103 | nohup /ckan-init.sh &> "${CKAN_LOG_DIR}"/ckan_init & 104 | } 105 | 106 | ckan_serve () { 107 | paster serve "${config}" 108 | } 109 | 110 | # Main section 111 | 112 | wait_for_services 113 | 114 | # If config file does not exist, create it 115 | if [ ! -e "${config}" ]; then 116 | write_config 117 | fi 118 | 119 | init_db 120 | 121 | harvesting 122 | 123 | ckan_configure 124 | 125 | ckan_serve 126 | 127 | exec "$@" 128 | -------------------------------------------------------------------------------- /ckan.ini: -------------------------------------------------------------------------------- 1 | # 2 | # CKAN - Pylons configuration 3 | # 4 | # These are some of the configuration options available for your CKAN 5 | # instance. Check the documentation in 'doc/configuration.rst' or at the 6 | # following URL for a description of what they do and the full list of 7 | # available options: 8 | # 9 | # http://docs.ckan.org/en/latest/maintaining/configuration.html 10 | # 11 | # The %(here)s variable will be replaced with the parent directory of this file 12 | # 13 | 14 | [DEFAULT] 15 | 16 | # WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* 17 | debug = false 18 | 19 | [server:main] 20 | use = egg:Paste#http 21 | host = 0.0.0.0 22 | port = 5000 23 | 24 | [app:main] 25 | use = egg:ckan 26 | full_stack = true 27 | cache_dir = /tmp/%(ckan.site_id)s/ 28 | beaker.session.key = ckan 29 | # HARVEST PLUGIN INSERT 30 | ckan.harvest.mq.type = redis 31 | ckan.harvest.mq.hostname = redis 32 | 33 | # This is the secret token that the beaker library uses to hash the cookie sent 34 | # to the client. `paster make-config` generates a unique value for this each 35 | # time it generates a config file. 36 | beaker.session.secret = 9yCxMlobgwzjltvAaLuThS5++ 37 | 38 | # `paster make-config` generates a unique value for this each time it generates 39 | # a config file. 40 | app_instance_uuid = bc7bff38-3c9b-4905-886b-6862bf7e9893 41 | 42 | # repoze.who config 43 | who.config_file = %(here)s/who.ini 44 | who.log_level = warning 45 | who.log_file = %(cache_dir)s/who_log.ini 46 | # Session timeout (user logged out after period of inactivity, in seconds). 47 | # Inactive by default, so the session doesn't expire. 48 | # who.timeout = 86400 49 | 50 | ## Database Settings 51 | sqlalchemy.url = postgresql://ckan:ckan@db:5432/ckan 52 | #sqlalchemy.url = postgresql://ckan:ckan@db/ckan 53 | 54 | # ckan.datastore.write_url = postgresql://ckan:ckan@db:5432/datastore 55 | # ckan.datastore.read_url = postgresql://datastore:datastore@db:5432/datastore 56 | 57 | #ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default 58 | #ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default 59 | 60 | # PostgreSQL' full-text search parameters 61 | # ckan.datastore.default_fts_lang = english 62 | # ckan.datastore.default_fts_index_method = gist 63 | 64 | ## Site Settings 65 | 66 | ckan.site_url = 67 | #ckan.use_pylons_response_cleanup_middleware = true 68 | 69 | ## Authorization Settings 70 | 71 | ckan.auth.anon_create_dataset = false 72 | ckan.auth.create_unowned_dataset = false 73 | ckan.auth.create_dataset_if_not_in_organization = false 74 | ckan.auth.user_create_groups = false 75 | ckan.auth.user_create_organizations = false 76 | ckan.auth.user_delete_groups = true 77 | ckan.auth.user_delete_organizations = true 78 | ckan.auth.create_user_via_api = false 79 | ckan.auth.create_user_via_web = false 80 | ckan.auth.roles_that_cascade_to_sub_groups = admin 81 | 82 | #DACAT-AP_IT 83 | ckanext.dcat.rdf.profiles = euro_dcat_ap it_dcat_ap 84 | ckanext.dcat.base_uri = http://localhost:5000 85 | 86 | ckanext.dcat.expose_subcatalogs = True 87 | 88 | ckanext.dcat.clean_tags = True 89 | 90 | ckanext.dcatapit.theme_group_mapping.file=/etc/ckan/default/theme_to_group.ini 91 | ckanext.dcatapit.nonconformant_themes_mapping.file = /etc/ckan/default/topics.json 92 | 93 | geonames.username = demo 94 | geonames.limits.countries = IT 95 | 96 | ## Search Settings 97 | 98 | ckan.site_id = dcatapit_docker_default 99 | solr_url=http://solr:8983/solr/ckan 100 | 101 | ## Redis Settings 102 | 103 | # URL to your Redis instance, including the database to be used. 104 | ckan.redis.url = redis://redis:6379/0 105 | #ckan.redis.url = http://redis:6379/0 106 | 107 | 108 | ## CORS Settings 109 | 110 | # If cors.origin_allow_all is true, all origins are allowed. 111 | # If false, the cors.origin_whitelist is used. 112 | # ckan.cors.origin_allow_all = true 113 | # cors.origin_whitelist is a space separated list of allowed domains. 114 | # ckan.cors.origin_whitelist = http://example1.com http://example2.com 115 | 116 | 117 | ## Plugins Settings 118 | 119 | # Note: Add ``datastore`` to enable the CKAN DataStore 120 | # Add ``datapusher`` to enable DataPusher 121 | # Add ``resource_proxy`` to enable resorce proxying and get around the 122 | # same origin policy 123 | # ckan.plugins = stats text_view image_view recline_view 124 | ckan.plugins = stats text_view image_view recline_view spatial_metadata spatial_query harvest ckan_harvester multilang multilang_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface dcatapit_pkg dcatapit_org dcatapit_config dcatapit_harvester dcatapit_ckan_harvester dcatapit_csw_harvester dcatapit_harvest_list dcatapit_subcatalog_facets dcatapit_theme_group_mapper 125 | 126 | ckan.spatial.srid = 4326 127 | #ckan.plugins = stats text_view image_view recline_view harvest ckan_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface dcatapit_pkg dcatapit_org dcatapit_config dcatapit_harvester 128 | #dcatapit_csw_harvester 129 | #multilang_harvester 130 | #csw_harvester 131 | #waf_harvester 132 | # doc_harvester 133 | 134 | 135 | 136 | # Define which views should be created by default 137 | # (plugins must be loaded in ckan.plugins) 138 | ckan.views.default_views = image_view text_view recline_view 139 | 140 | # Customize which text formats the text_view plugin will show 141 | #ckan.preview.json_formats = json 142 | #ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss 143 | #ckan.preview.text_formats = text plain text/plain 144 | 145 | # Customize which image formats the image_view plugin will show 146 | #ckan.preview.image_formats = png jpeg jpg gif 147 | 148 | ## Front-End Settings 149 | ckan.site_title = CKAN 150 | ckan.site_logo = /base/images/ckan-logo.png 151 | ckan.site_description = 152 | ckan.favicon = /base/images/ckan.ico 153 | ckan.gravatar_default = identicon 154 | ckan.preview.direct = png jpg gif 155 | ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json 156 | ckan.display_timezone = server 157 | 158 | # package_hide_extras = for_search_index_only 159 | #package_edit_return_url = http://another.frontend/dataset/ 160 | #package_new_return_url = http://another.frontend/dataset/ 161 | #ckan.recaptcha.version = 1 162 | #ckan.recaptcha.publickey = 163 | #ckan.recaptcha.privatekey = 164 | #licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json 165 | # ckan.template_footer_end = 166 | 167 | 168 | ## Internationalisation Settings 169 | ckan.locale_default = it 170 | ckan.locale_order = it de fr en pt_BR ja cs_CZ ca es el sv sr sr@latin no sk fi ru pl nl bg ko_KR hu sa sl lv 171 | ckan.locales_offered = it de fr en 172 | ckan.locales_filtered_out = it_IT 173 | 174 | ## Feeds Settings 175 | 176 | ckan.feeds.authority_name = 177 | ckan.feeds.date = 178 | ckan.feeds.author_name = 179 | ckan.feeds.author_link = 180 | 181 | ## Storage Settings 182 | 183 | #ckan.storage_path = /var/lib/ckan 184 | #ckan.max_resource_size = 10 185 | #ckan.max_image_size = 2 186 | 187 | ## Datapusher settings 188 | 189 | # Make sure you have set up the DataStore 190 | 191 | #ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet 192 | #ckan.datapusher.url = http://127.0.0.1:8800/ 193 | #ckan.datapusher.assume_task_stale_after = 3600 194 | 195 | # Resource Proxy settings 196 | # Preview size limit, default: 1MB 197 | #ckan.resource_proxy.max_file_size = 1048576 198 | # Size of chunks to read/write. 199 | #ckan.resource_proxy.chunk_size = 4096 200 | 201 | ## Activity Streams Settings 202 | 203 | #ckan.activity_streams_enabled = true 204 | #ckan.activity_list_limit = 31 205 | #ckan.activity_streams_email_notifications = true 206 | #ckan.email_notifications_since = 2 days 207 | ckan.hide_activity_from_users = %(ckan.site_id)s 208 | 209 | 210 | ## Email settings 211 | 212 | #email_to = errors@example.com 213 | #error_email_from = ckan-errors@example.com 214 | #smtp.server = localhost 215 | #smtp.starttls = False 216 | #smtp.user = username@example.com 217 | #smtp.password = your_password 218 | #smtp.mail_from = 219 | 220 | 221 | ## Logging configuration 222 | [loggers] 223 | keys = root, ckan, ckanext 224 | 225 | [handlers] 226 | keys = console, file 227 | 228 | [formatters] 229 | keys = generic 230 | 231 | [logger_root] 232 | level = WARNING 233 | handlers = console, file 234 | 235 | [logger_ckan] 236 | level = INFO 237 | handlers = console, file 238 | qualname = ckan 239 | propagate = 0 240 | 241 | [logger_ckanext] 242 | level = DEBUG 243 | handlers = console, file 244 | qualname = ckanext 245 | propagate = 0 246 | 247 | [handler_console] 248 | class = StreamHandler 249 | args = (sys.stderr,) 250 | level = NOTSET 251 | formatter = generic 252 | 253 | [handler_file] 254 | class = logging.handlers.RotatingFileHandler 255 | formatter = generic 256 | level = NOTSET 257 | args = ("/var/log/ckan/ckan.log", "a", 20000000, 9) 258 | 259 | [formatter_generic] 260 | format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s 261 | 262 | -------------------------------------------------------------------------------- /data/config/topics.json: -------------------------------------------------------------------------------- 1 | { 2 | "data" : [ 3 | { 4 | "syn" : [ 5 | "AGRI", 6 | "agri", 7 | "agricoltura", 8 | "agricoltura-e-allevamento", 9 | "agricoltura-e-pesca", 10 | "agricoltura-e-sviluppo-rurale", 11 | "agricoltura-pesca-politiche-forestali-e-alimentari", 12 | "animali", 13 | "attività agricola", 14 | "bevande e zucchero", 15 | "coltivazione di terreni agricoli", 16 | "foresta", 17 | "industria agro-alimentare", 18 | "mezzo di produzione agricola", 19 | "orientamento produttivo agricolo", 20 | "pesca", 21 | "politica agricola", 22 | "prodotto agricolo trasformato", 23 | "prodotto alimentare", 24 | "prodotto animale", 25 | "prodotto vegetale", 26 | "produzione e strutture agricole", 27 | "settore-primario", 28 | "silvicoltura e prodotti alimentari", 29 | "tecnologia alimentare", 30 | "Agricoltura, pesca, silvicoltura e prodotti alimentari" 31 | ], 32 | "name" : "Agricoltura, pesca, silvicoltura e prodotti alimentari" 33 | }, 34 | { 35 | "syn" : [ 36 | "ECON", 37 | "econ", 38 | "analisi economica", 39 | "assicurazioni", 40 | "attivita-produttive", 41 | "bilanci", 42 | "bilancio", 43 | "chimica", 44 | "commercializzazione", 45 | "commercio internazionale", 46 | "commercio-e-consumatori", 47 | "concorrenza", 48 | "consumo", 49 | "contabilità nazionale", 50 | "crescita economica", 51 | "distribuzione commerciale", 52 | "economia", 53 | "economia monetaria", 54 | "economia-finanze", 55 | "economia-finanze-e-tributi", 56 | "edilizia e lavori pubblici", 57 | "elettronica ed elettrotecnica", 58 | "finanza-e-controlli", 59 | "finanze pubbliche e politica di bilancio", 60 | "fiscalità", 61 | "forma giuridica di società", 62 | "gestione amministrativa", 63 | "gestione contabile-", 64 | "imprese", 65 | "industria del cuoio e dei prodotti tessili", 66 | "industria del legno", 67 | "industria meccanica", 68 | "industrie varie", 69 | "investimenti e finanziamenti", 70 | "istituti finanziari e di credito", 71 | "lavoro", 72 | "lavoro-e-formazione", 73 | "libera circolazione dei capitali", 74 | "metallurgia e siderurgia", 75 | "organizzazione aziendale", 76 | "politica commerciale", 77 | "politica economica", 78 | "politica tariffaria", 79 | "politiche e strutture industriali", 80 | "prezzi", 81 | "regione e politica regionale", 82 | "relazioni monetarie", 83 | "sardegna-turismo", 84 | "scambi economici", 85 | "struttura economica", 86 | "sviluppo-economico-formazione-e-lavoro", 87 | "terzo-settore", 88 | "tipo d'impresa", 89 | "turismo", 90 | "Economia e finanze" 91 | ], 92 | "name" : "Economia e finanze" 93 | }, 94 | { 95 | "syn" : [ 96 | "EDUC", 97 | "educ", 98 | "comunicazione", 99 | "conoscenza", 100 | "cultura", 101 | "cultura e religione", 102 | "cultura e sport", 103 | "culturaeturismo", 104 | "documentazione", 105 | "educazione-e-istruzione", 106 | "informatica e trattamento dei dati", 107 | "informazione ed elaborazione dell'informazione", 108 | "insegnamento", 109 | "istruzione", 110 | "istruzione1", 111 | "istruzione-cultura-e-sport", 112 | "istruzione-e-lavoro", 113 | "istruzione-e-sport", 114 | "istruzione-formazione-e-diritto-allo-studio", 115 | "organizzazione scolastica", 116 | "ravenna-2019", 117 | "ricerca-e-innovazione", 118 | "sardegna-digital-library", 119 | "scuola", 120 | "sport", 121 | "turismo-e-sport", 122 | "turismo-e-tempo-libero", 123 | "turismo-sport-e-tempo-libero", 124 | "vita sociale - tempo libero", 125 | "Istruzione, cultura e sport" 126 | ], 127 | "name" : "Istruzione, cultura e sport" 128 | }, 129 | { 130 | "syn" : [ 131 | "ener", 132 | "energia", 133 | "politica energetica", 134 | "industria carboniera e mineraria", 135 | "industria petrolifera", 136 | "industria nucleare ed elettrica", 137 | "energia dolce", 138 | "Energia" 139 | ], 140 | "name" : "Energia" 141 | }, 142 | { 143 | "syn" : [ 144 | "ENVI", 145 | "envi", 146 | "ambiente", 147 | "ambiente naturale", 148 | "ambiente-e-meteo", 149 | "ambiente-e-territorio", 150 | "ambientenaturale", 151 | "cartografia", 152 | "cat-meteo", 153 | "clima", 154 | "dati-ambientali", 155 | "dati-geografici", 156 | "degrado ambientale", 157 | "difesa-suolo", 158 | "foreste-e-parchi", 159 | "geodatabase", 160 | "geoportale-sardegna", 161 | "gestione-del-territorio", 162 | "meteo", 163 | "pianificazione-territoriale-comunale", 164 | "politica dell'ambiente", 165 | "superfici-territoriali", 166 | "territorio", 167 | "territorio-ambiente", 168 | "Ambiente" 169 | ], 170 | "name" : "Ambiente" 171 | }, 172 | { 173 | "syn" : [ 174 | "GOVE", 175 | "gove", 176 | "agenda-digitale-dell-umbria", 177 | "amministrazione", 178 | "amministrazione-comunale", 179 | "amministrazione-trasparente", 180 | "conti-pubblici-territoriali", 181 | "costruzione europea", 182 | "diritto dell'Unione Europea", 183 | "fesr2007-2013", 184 | "finanze dell'Unione europea", 185 | "government-e-settore-pubblico", 186 | "governo-e-settore-pubblico", 187 | "istituzioni dell'Unione europea e funzione pubblica europea", 188 | "lavori parlamentari", 189 | "opere-pubbliche", 190 | "Parlamento", 191 | "partito politico", 192 | "por-fesr-2014-2020", 193 | "potere esecutivo e amministrazione pubblica", 194 | "procedura elettorale e voto", 195 | "pubblica-amministrazione", 196 | "quadro politico", 197 | "Trasparenza", 198 | "urbanistica", 199 | "vita politica e sicurezza pubblica", 200 | "Governo e settore pubblico" 201 | ], 202 | "name" : "Governo e settore pubblico" 203 | }, 204 | { 205 | "syn" : [ 206 | "HEAL", 207 | "heal", 208 | "promozione-della-salute", 209 | "salute", 210 | "salute-e-sanita", 211 | "salute-e-sicurezza", 212 | "sanita", 213 | "sanita-e-servizi-sanitari", 214 | "Salute" 215 | ], 216 | "name" : "Salute" 217 | }, 218 | { 219 | "syn" : [ 220 | "INTR", 221 | "intr", 222 | "difesa", 223 | "equilibrio internazionale", 224 | "Nazioni Unite", 225 | "organizzazioni europee", 226 | "organizzazioni extraeuropee", 227 | "organizzazioni mondiali", 228 | "organizzazioni non governative", 229 | "politica di cooperazione", 230 | "politica internazionale", 231 | "Tematiche internazionali" 232 | ], 233 | "name" : "Tematiche internazionali" 234 | }, 235 | { 236 | "syn" : [ 237 | "JUST", 238 | "just", 239 | "diritti e libertà", 240 | "diritto civile", 241 | "diritto internazionale", 242 | "diritto penale", 243 | "emergenza-e-sicurezza", 244 | "fonti e branche del diritto", 245 | "giustizia", 246 | "Giustizia", 247 | "giustizia-e-sicurezza", 248 | "organizzazione della giustizia", 249 | "Sicurezza", 250 | "sistema giuridico e sicurezza pubblica", 251 | "Giustizia, sistema giuridico e sicurezza pubblica" 252 | ], 253 | "name" : "Giustizia, sistema giuridico e sicurezza pubblica" 254 | }, 255 | { 256 | "syn" : [ 257 | "REGI", 258 | "regi", 259 | "Africa", 260 | "America", 261 | "Asia-Oceania", 262 | "Europa", 263 | "geografia economica", 264 | "geografia politica", 265 | "paesi e territori d'oltremare", 266 | "regioni degli Stati membri dell'Unione europea", 267 | "Regioni e città", 268 | "regioni-e-citta", 269 | "Regioni e città" 270 | ], 271 | "name" : "Regioni e città" 272 | }, 273 | { 274 | "syn" : [ 275 | "SOCI", 276 | "soci", 277 | "amministrazione e remunerazione del personale", 278 | "condizioni e organizzazioni del lavoro", 279 | "dati-demografici", 280 | "dati-statistici", 281 | "dati-urbanistici", 282 | "demografia", 283 | "demografia e popolazione", 284 | "elettorale", 285 | "elezioni", 286 | "elezioni-e-attivita-politica", 287 | "famiglia", 288 | "incidenti-stradali", 289 | "lavoro-e-politiche-sociali", 290 | "mercato del lavoro", 291 | "migrazione", 292 | "occupazione", 293 | "politica", 294 | "popolazione-e-societa", 295 | "popolazione-e-sviluppo-demografico", 296 | "protezione sociale", 297 | "protezione-civile", 298 | "puglia-sociale", 299 | "quadro sociale", 300 | "rapporti di lavoro e diritto del lavoro", 301 | "sociale", 302 | "societa", 303 | "statistica", 304 | "urbanistica ed edilizia", 305 | "vita sociale", 306 | "welfare", 307 | "Popolazione e società" 308 | ], 309 | "name" : "Popolazione e società" 310 | }, 311 | { 312 | "syn" : [ 313 | "TECH", 314 | "tech", 315 | "produzione", 316 | "ricerca e proprietà intellettuale", 317 | "scienza-e-tecnologia", 318 | "scienze naturali e applicate", 319 | "scienze umane", 320 | "tecnologia e regolamentazione tecnica", 321 | "Scienza e tecnologia" 322 | ], 323 | "name" : "Scienza e tecnologia" 324 | }, 325 | { 326 | "syn" : [ 327 | "TRAN", 328 | "tran", 329 | "infrastrutture-e-trasporti", 330 | "mobilita", 331 | "mobilita-e-sicurezza", 332 | "mobilita-e-trasporti", 333 | "organizzazione dei trasporti", 334 | "politica dei trasporti", 335 | "trasporti", 336 | "trasporti aerei e spaziali", 337 | "trasporti marittimi e fluviali", 338 | "trasporti terrestri", 339 | "viabilita", 340 | "Trasporti" 341 | ], 342 | "name" : "Trasporti" 343 | } 344 | ], 345 | "meta" : { 346 | "authority_file_src" : "http://publications.europa.eu/resource/authority/data-theme/", 347 | "vocabulary" : "topics", 348 | "property" : "dcat:theme" 349 | } 350 | } 351 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CKAN-IT - The Italian distribution 2 | 3 | [![Data and open data on forum.italia.it](https://img.shields.io/badge/Forum-Dati%20e%20open%20data-blue.svg)](https://forum.italia.it/c/dati) 4 | [![Join the #pdnd-ckan channel](https://img.shields.io/badge/Slack%20channel-%23pdnd--ckan-blue.svg?logo=slack)](https://developersitalia.slack.com/messages/CMX9ZDPK3) 5 | [![Get invited](https://slack.developers.italia.it/badge.svg)](https://slack.developers.italia.it/) 6 | 7 | [CKAN](https://ckan.org/) is a powerful data management system that makes data accessible – by providing tools to streamline publishing, sharing, finding and using data. This project provides everything you need to run CKAN plus a set of extensions for supporting Italian open data in a set of Docker images. 8 | 9 | Any Italian public institution that wants to publish its data in an open format should follow these guidelines: ["Linee Guida Nazionali per la Valorizzazione del Patrimonio Informativo Pubblico"](https://www.dati.gov.it/sites/default/files/LG2016_0.pdf). Technical details and best practices for data catalogues development and management are contained in these guidelines: ["Linee guida per i cataloghi dati"](https://docs.italia.it/italia/daf/linee-guida-cataloghi-dati-dcat-ap-it/it/stabile/). Open data published by Italian public institutions should be compliant to the [national metadata profile called DCAT-AP_IT](https://www.dati.gov.it/content/dcat-ap-it-v10-profilo-italiano-dcat-ap-0). 10 | 11 | CKAN-IT is the Italian official CKAN distribution [packaged with plugins and external components that ensure the compliance with DCAT_AP-IT](#ckan-268-extensions-reference) and all the official guidelines mentioned above. Docker technology facilitates installation and deploy in production-ready environments. All third-party repository containing source code of components and plugins are mirrored under /italia Github organization, but maintained by original maintainer and community (ie. CKAN core, solr, postgresql, redis, and ckanext-harvest and -dcat). Only three plugins are directly developed and maintained within CKAN-IT project: ckanext-spatial (fork of the [official one](https://github.com/ckan/ckanext-spatial)), -multilang, and -dcatapit. Read below for more details. 12 | 13 | ## Tools references 14 | 15 | The tools used in this repository are 16 | 17 | * [CKAN](https://ckan.org/) 18 | * [Docker](https://www.docker.com/) 19 | 20 | ## Main components 21 | 22 | * **CKAN** version 2.6.8 with the extensions listed at the end of this document (see [italia/ckan](https://github.com/italia/ckan)). 23 | 24 | * **Solr** version 6.2 packaged for CKAN and with some customizations (see [italia/ckan-it-solr](https://github.com/italia/ckan-it-solr)). 25 | 26 | * **PostgreSQL** version 10.1, modified for CKAN (see https://hub.docker.com/r/ckan/postgresql/tags, tag *latest*). 27 | 28 | * **Redis** version 5.0.5, pulled in as a dependency from its [official Docker repository](https://hub.docker.com/_/redis). 29 | 30 | ## Plugins references 31 | 32 | Maintained plugins: 33 | * [spatial](https://github.com/italia/ckanext-spatial/) 34 | * [multilang](https://github.com/italia/ckanext-multilang/) 35 | * [dcatapit](https://github.com/italia/ckanext-dcatapit/) 36 | 37 | Official third-party plugins: 38 | * [harvest](https://github.com/italia/ckanext-harvest/) 39 | * [dcat](https://github.com/italia/ckanext-dcat/) 40 | 41 | ## How to run CKAN-IT 42 | 43 | In this repository, CKAN and its related tools are redistributed as a set of Docker containers interacting with one each other. 44 | 45 | The `Dockerfile` and the `docker-compose.yml` files are in the root of this repository. 46 | 47 | > NOTE: the `docker-compose.yml` file sets different environment variables that could be used to adapt and customized many platform functionalities, read more in "Environment variables" section below. 48 | 49 | If you want a CKAN-IT instance up and running in a couple of minutes, follow these steps. 50 | 51 | 1. Create and enter an empty folder: `mkdir ckan-it && cd ckan-it/` (or use the name you prefer) 52 | 2. Download the `docker-compose.yml` from [here](https://raw.githubusercontent.com/italia/ckan-it/master/docker-compose.yml) 53 | 3. Pull and run all containers: `docker-compose up -d` 54 | 55 | After a while you can open the CKAN-IT home [http://localhost:5000](http://localhost:5000) and login with the provided credentials. 56 | You can follow the log stream running `docker-compose logs -f` (then ctrl+c to exit). 57 | 58 | The following default credentials can be used to access the portal (you should change them after the first login). 59 | 60 | ``` 61 | Username: ckanadmin 62 | Password: ckanpassword 63 | ``` 64 | 65 | If you only want to run a CKAN-IT instance and use it to manage and publish your own data, you can stop here. 66 | In a production environment you can install and setup a proxy server in front of CKAN-IT with https support. 67 | 68 | > WARNING: all data are stored in [Docker named volumes](https://success.docker.com/article/different-types-of-volumes)! In a production environment you should mount these volumes on local folders updating the [docker-compose configuration](https://docs.docker.com/compose/compose-file/compose-file-v2/#volumes) accordingly. 69 | 70 | To bring down and remove the containers use `docker-compose down`. 71 | 72 | ## How to build and test CKAN-IT 73 | 74 | If you want to build local images instead of pull them from Dockerhub, ie. for testing pourpose, you need some extra steps. 75 | 76 | 1. Clone this repo: `git clone https://github.com/italia/ckan-it.git` (if you want to clone the repo in a folder other than `ckan-it/` add the name you want after the previous command, ie. `git clone https://github.com/italia/ckan-it.git my_custom_folder`) 77 | 2. Enter the created folder: `cd ckan-it/` (or the name you have chosen in previous step, ie. `cd my_custom_folder/`) 78 | 3. Change working branch if needed: `git checkout branch-name` 79 | 4. Initialize submodules: `git submodule update --init --recursive` 80 | 5. Build images: `docker-compose -f docker-compose.yml -f docker-compose.build.yml build` 81 | 6. Run all containers using built images: `docker-compose -f docker-compose.yml -f docker-compose.build.yml up -d` (if you want to check logs run `docker-compose logs -f`) 82 | 83 | ### A notice about CKAN customization 84 | 85 | This project brings together many components and plugins in a set of Docker images to facilitates installation and deploy. If you already have a running instance of CKAN or if you want to build a custom distribution from scratch you can install and use [single plugins](#plugins-references) following the [official documentation](https://docs.ckan.org/en/2.6/). 86 | 87 | ## CKAN-IT harvesting (optional) 88 | 89 | CKAN-IT can acts also as an aggregator of data sources, harvesting dataset metadata from external sources. 90 | If you want to import data from external sources, follow these additional steps. 91 | 92 | > WARNING: note that if `CKAN_HARVEST` variable in docker-compose is not set to `"true"` no organizations and sources are initially loaded (see below), so you must use the GUI to manually add new organizations and sources of your choice before next steps. 93 | 94 | 1. Browse to [http://localhost:5000/harvest](http://localhost:5000/harvest) to check all available sources or add new sources 95 | 2. Identify the name of the CKAN container with `docker container ls` (ie. `italia-ckan-it`) and run the following command: `docker exec -it italia-ckan-it /ckan-harvest.sh` 96 | 97 | You can see logs during harvesting import with following command: `docker-compose logs -f`. 98 | You can find more logs in `/var/log/ckan` folder inside the container. 99 | 100 | ### Run CKAN-IT periodic harvesting 101 | 102 | Schedule a CRON job on the host machine to run the `/ckan-harvest.sh` script at the root of the file system of the CKAN container. 103 | 104 | How to do this really depends on how you run the containers. When running containers with docker-compose for instance we did this by getting the container id and using `docker-exec` to run a command inside the container, as follows: 105 | `docker exec -it italia-ckan-it /ckan-harvest.sh 2>&1 /var/log/periodic-harvest.out` 106 | 107 | So you can schedule a periodic run of the above script, ie. every hour, with CRON on the host machine and save logs. 108 | 109 | ### Pre-load organizations and sources 110 | 111 | The [italia/public-opendata-sources](https://github.com/italia/public-opendata-sources) repository contains all sources harvested by the national catalog of the [Piattaforma Digitale Nazionale Dati (PDND) - previously DAF](https://pdnd.italia.it/). 112 | 113 | If you want to import all the official sources provided, simply run CKAN-IT setting the environment variable `CKAN_HARVEST="true"`, 114 | ie. `docker-compose -f docker-compose.yml -f docker-compose.harvest.yml up -d`. 115 | 116 | If you want to include them (and others of your choice) in built images (ie. for testing purpose), follow these additional steps: 117 | 118 | 1. Check if `data/init/harvesters` folder exists and fill them with the content of `https://github.com/italia/public-opendata-sources` (or whatever you want, but be sure that folders and json schemas are the same) 119 | 2. Build images: `docker-compose -f docker-compose.yml -f docker-compose.build.yml build` 120 | 3. Add `CKAN_HARVEST="true"` environment variable to the ckan service in `docker-compose.yml` (ie. see `docker-compose.harvest.yml`) 121 | 4. Run all containers using built images: `docker-compose -f docker-compose.yml -f docker-compose.build.yml up -d` (if you want to check logs run `docker-compose logs -f`) 122 | 5. Wait for organizations and harvest sources loading, then run `docker exec -it italia-ckan-it /ckan-harvest.sh` 123 | 6. Follow previous section to setup a periodic harvesting 124 | 125 | ### How to export your harvesting sources 126 | 127 | Read more [here](https://github.com/italia/public-opendata-sources#how-to-export-your-resources). 128 | 129 | ## Environment variables 130 | 131 | The following environment variables are mandatory and should be set in order to deploy CKAN-IT. The `docker-compose.yml` file in this repository applies some exemplar values, to be used for demos and local tests. 132 | 133 | ### General variables 134 | 135 | * CKAN_DEBUG *(format: {"true"|"false"})* - Whether to activate or not the debug log messages. It should always be false for production environments. 136 | 137 | * CKAN_HARVEST *(format: {"true"|"false"})* - Whether to activate or not the built-in harvesters. It should be false if you want to only build your own catalog and not harvest external sources. 138 | 139 | * CKAN_SITE_URL - The base URL of your CKAN-IT deployment. 140 | 141 | * CKAN_ADMIN_EMAIL - The email address of the local admin user. 142 | 143 | * CKAN_ADMIN_USERNAME - The user name of the local admin user. 144 | 145 | * CKAN_ADMIN_PASSWORD - The password of the local admin user. 146 | 147 | ### PostgreSQL variables 148 | 149 | * CKAN_DB_HOST - The host name of the CKAN PostgreSQL database. 150 | 151 | * CKAN_DB_PORT - The port of the CKAN PostgreSQL database. 152 | 153 | * CKAN_DB_USER - The user name of the CKAN PostgreSQL database. 154 | 155 | * PGPASSWORD - The password of the CKAN PostgreSQL database. 156 | 157 | * CKAN_SQLALCHEMY_URL *(format: {postgresql://{CKAN_DB_USER}:{PGPASSWORD}@{CKAN_DB_HOST}:{CKAN_DB_PORT}/})* - The connection string to your PostgreSQL database. 158 | 159 | ### Redis variables 160 | 161 | * CKAN_REDIS_HOST - The host name of your Redis service. 162 | 163 | * CKAN_REDIS_PORT - The port of your Redis service. 164 | 165 | * CKAN_REDIS_URL *(format: redis://{CKAN_REDIS_HOST}:/{CKAN_REDIS_PORT})* - The full address of the Redis service. 166 | 167 | ### Solr variables 168 | 169 | * CKAN_SOLR_HOST - The host name of the Solr service. 170 | 171 | * CKAN_SOLR_PORT - The port of the Solr service. 172 | 173 | * CKAN_SOLR_URL *(format: http://{CKAN_SOLR_HOST}:{CKAN_SOLR_PORT}/solr/ckan)* - The full URL of the Solr service. 174 | 175 | ## CKAN 2.6.8 extensions reference 176 | 177 | - stats 178 | - view 179 | - text_view 180 | - image_view 181 | - recline_view 182 | - datastore 183 | - [spatial](https://github.com/italia/ckanext-spatial/) (tag 2.6.8-2) 184 | - spatial_metadata 185 | - spatial_query 186 | - [harvest](https://github.com/italia/ckanext-harvest/) (tag v1.1.1) 187 | - ckan_harvester 188 | - [multilang](https://github.com/italia/ckanext-multilang/) (tag 2.6.8-2) 189 | - multilang_harvester 190 | - [dcat](https://github.com/italia/ckanext-dcat/) (tag v0.0.9) 191 | - dcat_rdf_harvester 192 | - dcat_json_harvester 193 | - dcat_json_interface 194 | - [dcatapit](https://github.com/italia/ckanext-dcatapit/) (tag 2.6.8-2) 195 | - dcatapit_pkg 196 | - dcatapit_org 197 | - dcatapit_config 198 | - dcatapit_harvester 199 | - dcatapit_csw_harvester 200 | - dcatapit_harvest_list 201 | - dcatapit_subcatalog_facets 202 | 203 | ## How to contribute 204 | 205 | Contributions are welcome. Feel free to open issues and submit a pull request at any time! 206 | --------------------------------------------------------------------------------