sysadmin/sprpcb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

repo inicializado

Browse Source
This commit is contained in:
Juan Burgos
2026-03-17 00:48:20 -04:00
parent 157c93736c
commit 717cc11a03
33 changed files with 9416 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
arches/Dockerfile Normal file
View File

@@ -0,0 +1,249 @@
FROM ubuntu:22.04
# Start with Ubuntu 22
USER root
# Set environment variables to make installations non-interactive
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ="America/La_Paz"
# Update package lists and install basic packages first
RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
wget \
gnupg \
lsb-release \
software-properties-common \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Add Python 3.12 repository and install Python packages
RUN add-apt-repository ppa:deadsnakes/ppa \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
python3.12 \
python3.12-dev \
python3.12-venv \
python3-pip \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Install system dependencies
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
make \
gcc \
build-essential \
mime-support \
libgdal-dev \
dos2unix \
nano \
git \
postgresql-client-14 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# set environment variables
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONUNBUFFERED 1
# Set up a virual environment to use for all later commands.
RUN python3.12 -m venv /opt/venv \
&& . /opt/venv/bin/activate \
&& pip install --upgrade pip
# Make sure we use the virtualenv:
ENV PATH="/opt/venv/bin:$PATH"
## Setting default environment variables
ARG ARCHES_ROOT
# The name of the arches project
ARG ARCHES_PROJECT
# Project specific paths
ARG APP_ROOT
ARG APP_COMP_FOLDER
ARG UPLOADED_FILES_FOLDER
ARG ARCHES_PIP_VERSION
# settings_local.py provides the DB credentials, etc. to the Arches project.
ENV SETTINGS_PATH=${APP_COMP_FOLDER}/settings.py
ENV SETTINGS_LOCAL_PATH=${APP_COMP_FOLDER}/settings_local.py
ENV CELERY_PATH=${APP_COMP_FOLDER}/celery.py
ENV URLS_PATH=${APP_COMP_FOLDER}/urls.py
ENV GUNICORN_CONFIG_PATH=${APP_COMP_FOLDER}/gunicorn_config.py
ENV ARCHES_DATA=${ARCHES_ROOT}/arches_data
ENV PACKAGE_PATH=${APP_COMP_FOLDER}/package.json
ENV ESLINT_CONFIG_MJS_PATH_USE=${APP_ROOT}/eslint.config.mjs
ENV UPLOADED_FILES_FOLDER=${UPLOADED_FILES_FOLDER}
ENV WHEELS=/wheels
ENV PYTHONUNBUFFERED=1
ENV PYTHONDONTWRITEBYTECODE 1
# Setup needed directories
RUN mkdir ${ARCHES_ROOT} && mkdir /var/log/supervisor && mkdir /var/log/celery
# Get ready to do some code installation
RUN apt-get update && apt-get install -y make software-properties-common
# Install packages required to run Arches
# Note that the ubuntu/debian package for libgdal1-dev pulls in libgdal1i, which is built
# with everything enabled, and so, it has a huge amount of dependancies (everything that GDAL
# support, directly and indirectly pulling in mysql-common, odbc, jp2, perl! ... )
# a minimised build of GDAL could remove several hundred MB from the container layer.
RUN set -ex \
&& RUN_DEPS=" \
build-essential \
libxml2-dev \
libproj-dev \
libjson-c-dev \
xsltproc \
docbook-xsl \
docbook-mathml \
libgdal-dev \
libpq-dev \
mime-support \
python3-dev \
postgresql-client-14 \
dos2unix \
wait-for-it \
vim \
" \
&& curl -sL https://deb.nodesource.com/setup_16.x | bash - \
&& curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg \
&& echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/postgresql.list \
&& apt-get update -y \
&& apt-get install -y --no-install-recommends $RUN_DEPS \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# ----------------------------------------------
# Do installs relating to Node
# ----------------------------------------------
# Set environment variables
ENV NODE_MAJOR=18
# Update package lists and install necessary packages
RUN apt-get update \
&& apt-get install -y \
ca-certificates \
curl \
gnupg \
&& mkdir -p /etc/apt/keyrings
# Add nodesource GPG key
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
# Add nodesource repository
RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
&& apt-get update
# Install Node.js
RUN apt-get install -y nodejs
# Clean up
RUN apt-get clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
WORKDIR ${ARCHES_ROOT}
RUN rm -rf /root/.cache/pip/*
RUN pip install --upgrade pip
RUN pip install --force-reinstall -U setuptools
# Install the current stable release of the Arches application and make an Arches project.
RUN pip install supervisor \
&& pip install pytz --upgrade \
&& pip install tzdata --upgrade \
&& pip install Redis \
&& pip install gunicorn \
&& pip install boto3==1.26 \
&& pip install django-storages==1.14.4 \
&& pip install arches==${ARCHES_PIP_VERSION}
# create a new Arches project
RUN mkdir -p /new_arches_proj
RUN arches-admin startproject ${ARCHES_PROJECT} --directory /new_arches_proj
RUN mv /new_arches_proj ${APP_ROOT}
# create the uploaded files folder
RUN mkdir -p ${UPLOADED_FILES_FOLDER}
# copia de archivos de idioma es
COPY ./arches/es /opt/venv/lib/python3.12/site-packages/arches/locale/es
RUN echo 'archivos de idioma es copiados'
# copy in local settings to our newly created Arches project.
WORKDIR ${APP_ROOT}
COPY ./arches/settings_local.py ${SETTINGS_LOCAL_PATH}
RUN sed -i 's/\r$//g' ${SETTINGS_LOCAL_PATH}
# copy the celery.py file into out archaes project. This hopefully makes the workers and beat work:
COPY ./arches/celery.py ${CELERY_PATH}
RUN sed -i 's/\r$//g' ${CELERY_PATH}
# copy the urls.py into our new Arches project. This is part of
# customization for internationalization
RUN echo "copied urls to ${URLS_PATH}";
COPY ./arches/urls.py ${URLS_PATH}
RUN sed -i 's/\r$//g' ${URLS_PATH}
# Copy the gunicorn_config.py file into our new Arches project. This is used
# for running Arches in production (not DEBUG mode)
# NOTE: We're NOT actually using this, because it will throw an error with urls.py
# So this is here for reference only in case someone wants to edit gunicorn_config.py
# and figure out how to make it work.
RUN echo "copy gunicorn_config.py to ${GUNICORN_CONFIG_PATH}";
COPY ./arches/gunicorn_config.py ${GUNICORN_CONFIG_PATH}
RUN sed -i 's/\r$//g' ${GUNICORN_CONFIG_PATH}
# Copy customized package until the issue with datatables.net is resolved.
# COPY ./arches/package.json ${PACKAGE_PATH}
# copy the celery supervisor
COPY /arches/conf.d/ ${APP_ROOT}/conf.d/
RUN chmod -R 700 ${APP_ROOT}/conf.d/
COPY /arches/arches_proj-supervisor.conf ${APP_ROOT}/arches_proj-supervisor.conf
RUN chmod -R 700 ${APP_ROOT}/arches_proj-supervisor.conf
RUN mkdir -p /var/log/supervisor
RUN mkdir -p /var/log/celery
# Now install NPM
WORKDIR ${APP_ROOT}
# RUN echo "NPM install....";
# RUN npm install
# Set some settings to make NPM less fussy
RUN npm config set cafile null
RUN npm config set strict-ssl false
# remove any node_modules that might have been installed by the arches install
RUN rm -rf ${APP_ROOT}/node_modules
RUN rm -f ${APP_ROOT}/package-lock.json
# Now do the NPM install
RUN npm install
# Make sure the entry point is available and lacks weird characters
# that don't work in a Linux OS
COPY /arches/entrypoint.sh ${APP_ROOT}/entrypoint.sh
RUN chmod -R 700 ${APP_ROOT}/entrypoint.sh &&\
dos2unix ${APP_ROOT}/entrypoint.sh
# Set default workdir
WORKDIR ${APP_ROOT}
ENTRYPOINT ["./entrypoint.sh"]
CMD ["run_arches"]
# Set default workdir
WORKDIR ${APP_ROOT}
# Expose port 8000 (Django server)
EXPOSE 8000
# Expose Webpack port
EXPOSE 8021
# Expose CouchDB port
EXPOSE 5984

4
arches/arches_data/README.md Normal file
View File

@@ -0,0 +1,4 @@
# arches_data
This folder will be mounted into the `arches` container so that it will be easier to load packages and exchange other files with the running Arches instance.
The `arches` container will expect to find a `packages` sub-directory inside the `arches_data` folder.

29
arches/arches_proj-supervisor.conf Normal file
View File

@@ -0,0 +1,29 @@
[unix_http_server]
file=/tmp/supervisor.sock ; path to your socket file
chmod=7770
[supervisord]
logfile=/var/log/supervisor/supervisord.log ; supervisord log file
logfile_maxbytes=50MB ; maximum size of logfile before rotation
logfile_backups=10 ; number of backed up logfiles
loglevel=info ; info, debug, warn, trace
pidfile=/var/run/supervisord.pid ; pidfile location
nodaemon=false ; run supervisord as a daemon
minfds=1024 ; number of startup file descriptors
minprocs=200 ; number of process descriptors
user=root ; defaults to whichever user is runs supervisor
childlogdir=/var/log/supervisor/ ; where child log files will live
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use unix:// schem for a unix sockets.
[include]
files=./conf.d/arches_proj-celeryd.conf
# While the above appears to work, and is needed for features like the Bulk
# Data manager, I can't seem to get celerybeat to work as below:
#
# files=./conf.d/arches_proj-celeryd.conf ./conf.d/arches_proj-celerybeat.conf

10
arches/celery.py Normal file
View File

@@ -0,0 +1,10 @@
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
ARCHES_PROJECT = os.getenv('ARCHES_PROJECT')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', f'{ARCHES_PROJECT}.settings')
app = Celery(ARCHES_PROJECT)
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()

22
arches/conf.d/arches_proj-celerybeat.conf Normal file
View File

@@ -0,0 +1,22 @@
; ================================
; celery beat supervisor
; ================================
[program:celerybeat]
command=python -m celery -A arches_proj.celery beat --schedule=/tmp/celerybeat-schedule --loglevel=INFO --pidfile=/tmp/celerybeat.pid
directory=/arches_app/arches_proj
user=root
numprocs=1
stdout_logfile=/var/log/celery/beat.log
stderr_logfile=/var/log/celery/beat.log
autostart=true
autorestart=true
startsecs=10
; Causes supervisor to send the termination signal (SIGTERM) to the whole process group.
stopasgroup=true
; if rabbitmq is supervised, set its priority higher
; so it starts first
priority=999

26
arches/conf.d/arches_proj-celeryd.conf Normal file
View File

@@ -0,0 +1,26 @@
; ==================================
; celery worker supervisor
; ==================================
[program:celery]
command=python -m celery -A arches_proj worker -l INFO
directory=/arches_app/arches_proj
user=root
numprocs=1
stdout_logfile=/var/log/celery/worker.log
stderr_logfile=/var/log/celery/worker.log
autostart=true
autorestart=true
startsecs=10
; Need to wait for currently executing tasks to finish at shutdown.
; Increase this if you have very long running tasks.
stopwaitsecs = 600
; Causes supervisor to send the termination signal (SIGTERM) to the whole process group.
stopasgroup=true
; Set Celery priority higher than default (999)
; so, if rabbitmq is supervised, it will start first.
priority=1000

435
arches/entrypoint.sh Normal file
View File

@@ -0,0 +1,435 @@
#!/bin/bash
# APP and YARN folder locations
APP_FOLDER=${APP_ROOT}
APP_COMP_FOLDER=${APP_COMP_FOLDER}
GUNICORN_CONFIG_PATH=${APP_COMP_FOLDER}/gunicorn_config.py
STATIC_ROOT=/static_root
STATIC_JS=${STATIC_ROOT}/js
WEBPACK_STATS_PATH=${APP_FOLDER}/webpack-stats.json
# Environmental Variables
export DJANGO_PORT=${DJANGO_PORT:-8000}
COUCHDB_URL=${COUCHDB_URL}
#Utility functions that check db status
wait_for_db() {
echo "Testing if database server is up..."
while [[ ! ${return_code} == 0 ]]
do
psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSERNAME} --dbname=postgres -c "select 1" >&/dev/null
return_code=$?
sleep 1
done
echo "Database server is up"
echo "Testing if Elasticsearch is up..."
while [[ ! ${return_code} == 0 ]]
do
curl -s "http://${ESHOST}:${ESPORT}/_cluster/health?wait_for_status=green&timeout=60s" >&/dev/null
return_code=$?
sleep 1
done
echo "Elasticsearch is up"
}
db_exists() {
echo "Checking if database "${PGDBNAME}" exists..."
count=`psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSERNAME} --dbname=postgres -Atc "SELECT COUNT(*) FROM pg_catalog.pg_database WHERE datname='${PGDBNAME}'"`
# Check if returned value is a number and not some error message
re='^[0-9]+$'
if ! [[ ${count} =~ $re ]] ; then
echo "Error: Something went wrong when checking if database "${PGDBNAME}" exists..." >&2;
echo "Exiting..."
exit 1
fi
# Return 0 (= true) if database exists
if [[ ${count} > 0 ]]; then
return 0
else
return 1
fi
}
#### Install
init_arches() {
echo "Checking if Arches project "${ARCHES_PROJECT}" exists..."
if [[ ! -d ${APP_FOLDER} ]] || [[ ! "$(ls ${APP_FOLDER})" ]]; then
echo ""
echo "----- Custom Arches project '${ARCHES_PROJECT}' does not exist. -----"
echo "----- Creating '${ARCHES_PROJECT}'... -----"
echo ""
cd ${APP_FOLDER}
echo "Sleep for a 45 seconds because elastic search seems to need the wait (a total hack)..."
sleep 45s;
arches-project create ${ARCHES_PROJECT}
run_setup_db
setup_couchdb
exit_code=$?
if [[ ${exit_code} != 0 ]]; then
echo "Something went wrong when creating your Arches project: ${ARCHES_PROJECT}."
echo "Exiting..."
exit ${exit_code}
fi
else
echo "Custom Arches project '${ARCHES_PROJECT}' exists."
wait_for_db
if db_exists; then
echo "Database ${PGDBNAME} already exists."
echo "Skipping Package Loading"
else
echo "Database ${PGDBNAME} does not exists yet."
run_setup_db
run_elastic_safe_migrations
setup_couchdb
fi
fi
}
# Setup Couchdb
setup_couchdb() {
echo "--- SKIP Creating couchdb system databases (not in V7, no Collector) ---"
# echo "Sleep for a 10 seconds because elastic search seems to need the wait (a total hack)..."
# sleep 10s;
# curl -X PUT ${COUCHDB_URL}/_users
# curl -X PUT ${COUCHDB_URL}/_global_changes
# curl -X PUT ${COUCHDB_URL}/_replicator
}
#### Misc
check_settings_local() {
# Make sure we have a settings_local in the proper location of the project
cd ${APP_COMP_FOLDER}
echo "The directory ${APP_COMP_FOLDER} contains:"
ls -l
echo "---------------------------------------------------------------"
}
#### Run commands
start_celery_supervisor() {
echo ""
echo "----- START CELERY SUPERVISOR -----"
echo ""
echo "Sleep 60s in the hope that arches_redis will be fully up and running..."
sleep 60s;
if [ -f "/tmp/supervisor.sock" ]; then
echo "The celery supervisor seems started, so why try to start it again? "
else
echo "The celery supervisor has yet to start, so we'll start it.."
cd ${APP_FOLDER}
wait-for-it arches_redis:6379 -t 120 && supervisord -c arches_proj-supervisor.conf
fi
}
run_createcachetable() {
echo ""
echo "----- RUNNING CREATE CACHETABLE -----"
echo ""
cd ${APP_FOLDER}
python3 manage.py createcachetable
}
run_elastic_safe_migrations() {
echo ""
echo "----- RUNNING DATABASE MIGRATIONS WITH ELASTIC CHECK -----"
echo ""
echo "Testing if Elasticsearch is up..."
while [[ ! ${return_code} == 0 ]]
do
curl -s "http://${ESHOST}:${ESPORT}/_cluster/health?wait_for_status=green&timeout=60s" >&/dev/null
return_code=$?
sleep 1
done
echo "Elasticsearch is up"
cd ${APP_FOLDER}
echo "Sleep for a 20 seconds because elastic search seems to need the wait (a total hack)..."
echo "We're running migrations in case the initial db setup failed because elasticsearch was still not quite ready"
sleep 20s;
echo "Now do Migrations..."
python3 manage.py migrate
}
run_make_migrations() {
echo ""
echo "----- RUNNING DATABASE MAKE MIGRATIONS -----"
echo ""
cd ${APP_FOLDER}
python manage.py makemigrations
}
run_migrations() {
echo ""
echo "----- RUNNING DATABASE MIGRATIONS -----"
echo ""
cd ${APP_FOLDER}
python manage.py migrate
}
run_es_reindex() {
echo ""
echo "----- RUNNING ELASTIC SEARCH (ES) REINDEX DATABASE -----"
echo ""
cd ${APP_FOLDER}
python3 manage.py es reindex_database
}
run_collect_static() {
echo ""
echo "----- RUNNING COLLECT STATIC -----"
echo ""
if [[ ${BUILD_PRODUCTION} == 'True' ]]; then
echo "Skipping collectstatic, hopefully buildproduction will do the trick..."
else
cd ${APP_FOLDER}
python3 manage.py collectstatic --noinput
fi
echo "---------------------------------------------------------------"
}
run_collect_static_nocheck() {
echo ""
echo "----- RUNNING COLLECT STATIC -----"
echo ""
cd ${APP_FOLDER}
python3 manage.py collectstatic --noinput
echo "---------------------------------------------------------------"
}
run_build_production() {
echo ""
echo "----- RUNNING BUILD PRODUCTION -----"
echo ""
if [[ ${BUILD_PRODUCTION} == 'True' ]]; then
# NOTE: Only do this if you have more than 8GB of system RAM. This will likely error out
# otherwise.
cd ${APP_FOLDER}
exec sh -c "npm run build_development"
else
echo "Skipping buildproduction because BUILD_PRODUCTION is not 'True' "
fi
echo "---------------------------------------------------------------"
}
run_setup_arches_setup_webpack() {
if [[ ! -d ${STATIC_JS} ]] || [[ ! "$(ls ${STATIC_JS})" ]]; then
cd ${APP_FOLDER}
echo "Starting Django development server"
python manage.py runserver 0.0.0.0:8000 &
echo "Running npm build and collectstatic"
npm run build_development && python manage.py collectstatic --noinput
else
echo "Webpack and Collectstatic for setup already completed.";
fi
RUNSERVER_PID=$(pgrep -f "manage.py runserver")
if [ -n "$RUNSERVER_PID" ]; then
echo "Killing manage.py runserver process with PID $RUNSERVER_PID"
kill -9 $RUNSERVER_PID
echo "Process $RUNSERVER_PID killed"
else
echo "No manage.py runserver process found"
fi
}
run_webpack() {
echo ""
echo "----- *** RUNNING WEBPACK SERVER *** -----"
echo ""
if [[ ${BUILD_PRODUCTION} == 'True' ]]; then
# NOTE: Only do this if you have more than 8GB of system RAM. This will likely error out
# otherwise.
echo "Running Webpack, hopefully the build_production thing will work!"
cd ${APP_FOLDER}
exec sh -c "npm run build_production"
else
cd ${APP_FOLDER}
echo "Do build_development."
echo "Running Webpack to do the NPM build_development thing."
exec sh -c "npm run build_development && python manage.py collectstatic --noinput"
fi
}
run_setup_webpack() {
# NOTE: We're deprecating this in favor of run_setup_arches_setup_webpack.
echo ""
echo "----- *** RUNNING WEBPACK SERVER FOR SETUP *** -----"
echo ""
echo "Check if the Arches app responds to http requests..."
while [[ ! ${return_code} == 0 ]]
do
curl -s "http://arches:8000" >&/dev/null
return_code=$?
sleep 5
done
echo "Arches app is now responding to http requests!"
sleep 5
# We're going to first check to see if we have anythin in the static_root/js folder.
# If we do, then we've run this already and can skip webpack and collect static.
if [[ ! -d ${STATIC_JS} ]] || [[ ! "$(ls ${STATIC_JS})" ]]; then
echo "We (apparently) have yet to run webpack and collectstatic. Do it now!";
run_webpack
else
echo "Webpack and Collectstatic for setup already completed.";
# exec sh -c "python manage.py collectstatic --noinput"
fi
}
run_list_static() {
echo ""
echo "----- VIEW COLLECTED STATIC -----"
echo ""
cd /static_root
ls
echo "---------------------------------------------------------------"
}
run_setup_db() {
echo ""
echo "----- RUNNING SETUP_DB -----"
echo ""
echo "Testing if Elasticsearch is up..."
while [[ ! ${return_code} == 0 ]]
do
curl -s "http://${ESHOST}:${ESPORT}/_cluster/health?wait_for_status=green&timeout=60s" >&/dev/null
return_code=$?
sleep 1
done
echo "Elasticsearch is up, pause for 10 secs to be sure."
sleep 10s;
echo "Now we should be safe to setup the database"
cd ${APP_FOLDER}
python3 manage.py setup_db --force
}
run_load_package() {
echo ""
echo "----- *** LOADING PACKAGE: ${ARCHES_PROJECT} *** -----"
echo ""
cd ${APP_FOLDER}
python3 manage.py packages -o load_package -s ${ARCHES_PROJECT}/pkg -db -dev -y
}
run_django_server() {
echo ""
echo "----- *** RUNNING DJANGO DEVELOPMENT SERVER *** -----"
echo ""
cd ${APP_FOLDER}
if [[ ${DJANGO_DEBUG} == 'True' ]]; then
echo "Running DEBUG mode Django"
exec sh -c "python3 manage.py runserver 0.0.0.0:${DJANGO_PORT}"
else
echo "Should run the production mode Arches Django via gunicorn via:"
# The GUNICORN_CONFIG_PATH breaks this, (errors in urls.py) so we'll just run it directly
# echo "gunicorn ${ARCHES_PROJECT}.wsgi:application --config ${GUNICORN_CONFIG_PATH}"
# exec sh -c "gunicorn ${ARCHES_PROJECT}.wsgi:application --config ${GUNICORN_CONFIG_PATH}"
echo "gunicorn -w 2 -b 0.0.0.0:${DJANGO_PORT} ${ARCHES_PROJECT}.wsgi:application --reload --timeout 3600"
exec sh -c "gunicorn -w 2 -b 0.0.0.0:${DJANGO_PORT} ${ARCHES_PROJECT}.wsgi:application --reload --timeout 3600"
fi
}
#### Main commands
run_arches() {
init_arches
run_elastic_safe_migrations
run_createcachetable
start_celery_supervisor
run_setup_arches_setup_webpack
run_django_server
}
#### Main commands
run_livereload() {
run_livereload_server
}
### Starting point ###
# trying not to use virtualenv???
# activate_virtualenv
# Use -gt 1 to consume two arguments per pass in the loop
# (e.g. each argument has a corresponding value to go with it).
# Use -gt 0 to consume one or more arguments per pass in the loop
# (e.g. some arguments don't have a corresponding value to go with it, such as --help ).
# If no arguments are supplied, assume the server needs to be run
if [[ $# -eq 0 ]]; then
wait_for_db
run_arches
fi
# Else, process arguments
echo "Full command: $@"
while [[ $# -gt 0 ]]
do
key="$1"
echo "Command: ${key}"
case ${key} in
run_arches)
wait_for_db
run_arches
;;
run_livereload)
run_livereload_server
;;
run_collect_static)
run_collect_static
;;
run_collect_static_nocheck)
run_collect_static_nocheck
;;
run_list_static)
run_list_static
;;
run_setup_arches_setup_webpack)
run_setup_arches_setup_webpack
;;
run_setup_webpack)
run_setup_webpack
;;
run_webpack)
run_webpack
;;
run_build_production)
run_build_production
;;
setup_arches)
start_celery_supervisor
wait_for_db
setup_arches
;;
run_make_migrations)
wait_for_db
run_make_migrations
;;
run_migrations)
wait_for_db
run_migrations
;;
run_es_reindex)
wait_for_db
run_es_reindex
;;
help|-h)
display_help
;;
*)
cd ${APP_FOLDER}
"$@"
exit 0
;;
esac
shift # next argument or value
done

BIN
arches/es/LC_MESSAGES/django.mo Normal file
View File

Binary file not shown.

6483
arches/es/LC_MESSAGES/django.po Normal file
View File

File diff suppressed because it is too large Load Diff

242
arches/gunicorn_config.py Normal file
View File

@@ -0,0 +1,242 @@
# Copied from:
# https://github.com/archesproject/arches/blob/dev/7.6.x/docker/gunicorn_config.py
#
# NOTE: This does NOT work and will throw errors with urls.py if used as is.
# Edit this to customize Gunicorn settings
import os
from django.core.exceptions import ImproperlyConfigured
def get_optional_env_variable(var_name):
try:
return os.environ[var_name]
except KeyError:
return None
#
# Server socket
#
# bind - The socket to bind.
#
# A string of the form: 'HOST', 'HOST:PORT', 'unix:PATH'.
# An IP is a valid HOST.
#
# backlog - The number of pending connections. This refers
# to the number of clients that can be waiting to be
# served. Exceeding this number results in the client
# getting an error when attempting to connect. It should
# only affect servers under significant load.
#
# Must be a positive integer. Generally set in the 64-2048
# range.
#
django_port = get_optional_env_variable("DJANGO_PORT")
listen_port = django_port or "8000"
bind = "0.0.0.0:" + listen_port
backlog = get_optional_env_variable("GUNICORN_BACKLOG") or 2048
#
# Worker processes
#
# workers - The number of worker processes that this server
# should keep alive for handling requests.
#
# A positive integer generally in the 2-4 x $(NUM_CORES)
# range. You'll want to vary this a bit to find the best
# for your particular application's work load.
#
# worker_class - The type of workers to use. The default
# sync class should handle most 'normal' types of work
# loads. You'll want to read
# http://docs.gunicorn.org/en/latest/design.html#choosing-a-worker-type
# for information on when you might want to choose one
# of the other worker classes.
#
# A string referring to a Python path to a subclass of
# gunicorn.workers.base.Worker. The default provided values
# can be seen at
# http://docs.gunicorn.org/en/latest/settings.html#worker-class
#
# worker_connections - For the eventlet and gevent worker classes
# this limits the maximum number of simultaneous clients that
# a single process can handle.
#
# A positive integer generally set to around 1000.
#
# timeout - If a worker does not notify the master process in this
# number of seconds it is killed and a new worker is spawned
# to replace it.
#
# Generally set to thirty seconds. Only set this noticeably
# higher if you're sure of the repercussions for sync workers.
# For the non sync workers it just means that the worker
# process is still communicating and is not tied to the length
# of time required to handle a single request.
#
# keepalive - The number of seconds to wait for the next request
# on a Keep-Alive HTTP connection.
#
# A positive integer. Generally set in the 1-5 seconds range.
#
workers = get_optional_env_variable("GUNICORN_WORKERS") or 2
worker_class = get_optional_env_variable("GUNICORN_WORKER_CLASS") or "sync"
worker_connections = get_optional_env_variable("GUNICORN_WORKER_CONNECTIONS") or 1000
timeout = get_optional_env_variable("GUNICORN_WORKER_TIMEOUT") or 30
keepalive = get_optional_env_variable("GUNICORN_KEEPALIVE") or 2
#
# spew - Install a trace function that spews every line of Python
# that is executed when running the server. This is the
# nuclear option.
#
# True or False
#
spew = get_optional_env_variable("GUNICORN_SPEW") or False
#
# Server mechanics
#
# daemon - Detach the main Gunicorn process from the controlling
# terminal with a standard fork/fork sequence.
#
# True or False
#
# raw_env - Pass environment variables to the execution environment.
#
# pidfile - The path to a pid file to write
#
# A path string or None to not write a pid file.
#
# user - Switch worker processes to run as this user.
#
# A valid user id (as an integer) or the name of a user that
# can be retrieved with a call to pwd.getpwnam(value) or None
# to not change the worker process user.
#
# group - Switch worker process to run as this group.
#
# A valid group id (as an integer) or the name of a user that
# can be retrieved with a call to pwd.getgrnam(value) or None
# to change the worker processes group.
#
# umask - A mask for file permissions written by Gunicorn. Note that
# this affects unix socket permissions.
#
# A valid value for the os.umask(mode) call or a string
# compatible with int(value, 0) (0 means Python guesses
# the base, so values like "0", "0xFF", "0022" are valid
# for decimal, hex, and octal representations)
#
# tmp_upload_dir - A directory to store temporary request data when
# requests are read. This will most likely be disappearing soon.
#
# A path to a directory where the process owner can write. Or
# None to signal that Python should choose one on its own.
#
daemon = get_optional_env_variable("GUNICORN_DAEMON") or False
raw_env = get_optional_env_variable("GUNICORN_RAW_ENV") or []
pidfile = get_optional_env_variable("GUNICORN_PIDFILE") or None
umask = get_optional_env_variable("GUNICORN_UMASK") or 0
user = get_optional_env_variable("GUNICORN_USER") or None
group = get_optional_env_variable("GUNICORN_GROUP") or None
tmp_upload_dir = get_optional_env_variable("GUNICORN_TMP_UPLOAD_DIR") or None
#
# Logging
#
# logfile - The path to a log file to write to.
#
# A path string. "-" means log to stdout.
#
# loglevel - The granularity of log output
#
# A string of "debug", "info", "warning", "error", "critical"
#
errorlog = get_optional_env_variable("GUNICORN_ERRORLOG") or "-"
loglevel = get_optional_env_variable("GUNICORN_LOGLEVEL") or "info"
accesslog = get_optional_env_variable("GUNICORN_ACCESSLOG") or "-"
access_log_format = (
get_optional_env_variable("GUNICORN_ACCESS_LOG_FORMAT")
or '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
)
#
# Process naming
#
# proc_name - A base to use with setproctitle to change the way
# that Gunicorn processes are reported in the system process
# table. This affects things like 'ps' and 'top'. If you're
# going to be running more than one instance of Gunicorn you'll
# probably want to set a name to tell them apart. This requires
# that you install the setproctitle module.
#
# A string or None to choose a default of something like 'gunicorn'.
#
proc_name = get_optional_env_variable("GUNICORN_PROC_NAME") or None
#
# Server hooks
#
# post_fork - Called just after a worker has been forked.
#
# A callable that takes a server and worker instance
# as arguments.
#
# pre_fork - Called just prior to forking the worker subprocess.
#
# A callable that accepts the same arguments as after_fork
#
# pre_exec - Called just prior to forking off a secondary
# master process during things like config reloading.
#
# A callable that takes a server instance as the sole argument.
#
def post_fork(server, worker):
server.log.info("Worker spawned (pid: %s)", worker.pid)
def pre_fork(server, worker):
pass
def pre_exec(server):
server.log.info("Forked child, re-executing.")
def when_ready(server):
server.log.info("Server is ready. Spawning workers")
def worker_int(worker):
worker.log.info("worker received INT or QUIT signal")
# get traceback info
import threading
import sys
import traceback
id2name = {th.ident: th.name for th in threading.enumerate()}
code = []
for threadId, stack in list(sys._current_frames().items()):
code.append("\n# Thread: %s(%d)" % (id2name.get(threadId, ""), threadId))
for filename, lineno, name, line in traceback.extract_stack(stack):
code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
if line:
code.append(" %s" % (line.strip()))
worker.log.debug("\n".join(code))
def worker_abort(worker):
worker.log.info("worker received SIGABRT signal")

118
arches/settings_local.py Normal file
View File

@@ -0,0 +1,118 @@
import os
from django.core.exceptions import ImproperlyConfigured
import ast
def get_env_variable(var_name):
msg = "Set the %s environment variable"
try:
return os.environ[var_name]
except KeyError:
error_msg = msg % var_name
raise ImproperlyConfigured(error_msg)
def get_optional_env_variable(var_name):
try:
return os.environ[var_name]
except KeyError:
return None
# options are either "PROD" or "DEV"
# (installing with Dev mode set gets you extra dependencies)
MODE = get_env_variable("DJANGO_MODE")
DEBUG = ast.literal_eval(get_env_variable("DJANGO_DEBUG"))
if not DEBUG:
# Some extra security settings for production deployments
SESSION_COOKIE_SECURE = True
# We could use the DOMAINS envinronment variable here, but
# since we're only supporting one domain and that's the same
# has DEPLOY_HOST that is used for the SSL CERT_PATH.
DEPLOY_HOST = get_env_variable("DEPLOY_HOST")
CSRF_TRUSTED_ORIGINS = [
f"https://{DEPLOY_HOST}",
]
# Set the APP_NAME here too, it may be useful for making the URLs
# work correctly when running gunicorn.
APP_NAME = get_env_variable("ARCHES_PROJECT")
DATABASES = {
"default": {
"ENGINE": "django.contrib.gis.db.backends.postgis",
"NAME": get_env_variable("PGDBNAME"),
"USER": get_env_variable("PGUSERNAME"),
"PASSWORD": get_env_variable("PGPASSWORD"),
"HOST": get_env_variable("PGHOST"),
"PORT": get_env_variable("PGPORT"),
"POSTGIS_TEMPLATE": "template_postgis",
}
}
ARCHES_NAMESPACE_FOR_DATA_EXPORT = get_env_variable("ARCHES_NAMESPACE")
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.redis.RedisCache",
"LOCATION": "redis://arches_redis:6379/1",
},
"user_permission": {
"BACKEND": "django.core.cache.backends.db.DatabaseCache",
"LOCATION": "user_permission_cache",
},
}
"""
Since we're using Docker, we can use Redis (even on a Windows OS). So, we
will comment out the RabbitMQ connection in favor of a Redis connection.
CELERY_BROKER_URL = "amqp://{}:{}@arches_rabbitmq:5672".format(
get_env_variable("RABBITMQ_USER"), get_env_variable("RABBITMQ_PASS")
)
"""
CELERY_BROKER_URL = "redis://@arches_redis:6379/0"
# NOTE: If you want to disable celery and workers, leave a blank string fo
# the CELERY_BROKER_URL as follows:
#
# CELERY_BROKER_URL = ""
# CANTALOUPE_HTTP_ENDPOINT = "http://{}:{}".format(get_env_variable("CANTALOUPE_HOST"), get_env_variable("CANTALOUPE_PORT"))
ELASTICSEARCH_HTTP_PORT = get_env_variable("ESPORT")
ELASTICSEARCH_HOSTS = [
{
"scheme": "http",
"host": get_env_variable("ESHOST"),
"port": int(ELASTICSEARCH_HTTP_PORT),
}
]
USER_ELASTICSEARCH_PREFIX = get_optional_env_variable("ELASTICSEARCH_PREFIX")
if USER_ELASTICSEARCH_PREFIX:
ELASTICSEARCH_PREFIX = USER_ELASTICSEARCH_PREFIX
ALLOWED_HOSTS = get_env_variable("DOMAIN_NAMES").split() + ['*']
USER_SECRET_KEY = get_optional_env_variable("DJANGO_SECRET_KEY")
if USER_SECRET_KEY:
# Make this unique, and don't share it with anybody.
SECRET_KEY = USER_SECRET_KEY
STATIC_ROOT = "/static_root"
LANGUAGE_CODE = 'es'
# Added for v7 internationalization demo
# Change these to match the languages you want to support
LANGUAGES = [
('es', ('Spanish')),
('en', ('English')),
]
# This does not work when using gunicorn
# SHOW_LANGUAGE_SWITCH = len(LANGUAGES) > 1
SHOW_LANGUAGE_SWITCH = False

22
arches/urls.py Normal file
View File

@@ -0,0 +1,22 @@
from django.conf import settings
from django.conf.urls.static import static
from django.conf.urls.i18n import i18n_patterns
from django.urls import include, path
urlpatterns = [
# project-level urls
]
# Ensure Arches core urls are superseded by project-level urls
urlpatterns.append(path('', include('arches.urls')))
# Adds URL pattern to serve media files during development
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
# Only handle i18n routing in active project. This will still handle the routes provided by Arches core and Arches applications,
# but handling i18n routes in multiple places causes application errors.
if settings.ROOT_URLCONF == __name__:
if settings.SHOW_LANGUAGE_SWITCH is True:
urlpatterns = i18n_patterns(*urlpatterns)
urlpatterns.append(path("i18n/", include("django.conf.urls.i18n")))