Merge branch 'dockerization' into 'DEV'

Dockerization

See merge request !25

.dockerignore

+# database file
+db.sqlite
+environment.sh.example
+
+# IDEs
+.vscode/
+.idea/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+example.py
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+*/db.sqlite3
+*/*/db.sqlite3
+
+# pyenv
+.python-version
+
+# Cloud configure
+clouds.yaml
+
+# Git
+.gitignore
+.gitmodules
+.git*
\ No newline at end of file

.env.example

+# This variables are interpreted automatically by the Docker Engine
+# And by the Django runtime
+
+# Django configuration
+DJANGO_SECRET=django_secret_value
+# settings.local for development and testing
+# settings.production for production deployment
+DJANGO_SETTINGS_MODULE=myproject.settings.local
+
+# If using other database than sqlite3 (only use sqlite for development!)
+# Provide the db credentials here
+DATABASE_PASSWORD=database_password
+DATABASE_USER=user
+DATABASE_NAME=user
+DATABASE_HOST=hostname_of_database
+
+# For channels communication and periodic tasks we need a redis server
+REDIS_HOST=redis_host
+
+# OpenStack credentials
+# In development we recommend using a DevStack installation
+# These settings displayed in an admin dashboard or
+# Can be downloaded from Horizon, as an admin user
+OS_AUTH_URL=keysone_auth_url
+OS_PROJECT_ID=project_id
+OS_PROJECT_NAME=project_name
+OS_USER_DOMAIN_NAME=domain_name
+OS_USER_DOMAIN_ID=user_domain_id
+OS_PROJECT_DOMAIN_ID=project_doamin_id
+OS_USERNAME=username
+OS_PASSWORD=password
+OS_REGION_NAME=region_name
+OS_INTERFACE=interface_type
+OS_IDENTITY_API_VERSION=identity_api_version
\ No newline at end of file

.gitignore

@@ -72,6 +72,7 @@ db.sqlite3
 #Pipfile.lock
 
 # Environments
+*.env
 .env
 .venv
 env/
@@ -83,3 +84,6 @@ environment.sh
 
 # Cloud configure
 clouds.yaml
+
+# Celery beat
+celerybeat-schedule

.gitlab-ci.yml

 image: python:3.6
 
 before_script:
+  - chmod +x ./setup_ci_env.sh
+  - ./setup_ci_env.sh
   - git submodule sync --recursive
   - git submodule update --init --recursive
 
+stages:
+  - testing
+  - build
+  - deploy
+
 flake8:
+  stage: testing
   script:
-    - pip install flake8
+    - python3.6 -m pip install flake8
     - flake8
 
 test:
+  stage: testing
+  script:
+    - docker build --tag recircle-backend-local:latest .
+    - docker run recircle-backend-local:latest pipenv run python manage.py test
+
+build_prod_image:
+  stage: build
   script:
-    - pip install pipenv
-    - pipenv install -d
-    - cd recircle
-    - pipenv run python manage.py test
+    - docker build -f ./Dockerfile.prod --tag recircle-backend:latest .
+
+# deploy needs sshpass installed on gitlab runner machince
+deploy_staging:
+  stage: deploy
+  script: # When the docker registry ready, pull images from it
+    - sshpass -p "$STAGING_SSH_PASS" ssh -o StrictHostKeyChecking=no cloud@vm.niif.cloud.bme.hu -p 17668 &&
+      git clone https://git.ik.bme.hu/RECIRCLE/portal.git &&
+      cd portal &&
+      git submodule update --init --recursive &&
+      docker-compose -f docker-compose.prod.yml up --build &&
+      exit
+  environment:
+    name: staging
+    url: http://vm.niif.cloud.bme.hu:20088
+  only:
+    - DEV
+    - master

Dockerfile

+FROM python:3.6
+WORKDIR /usr/cloud/portal 
+SHELL ["/bin/bash", "-c"]
+# Install pipenv
+RUN pip install pipenv
+# Open 8000 port
+EXPOSE 8000/tcp
+# Copy files necessary for install dependencies
+COPY Pipfile* .env ./
+# This env variable needed to pipenv find .env file
+ENV PIPENV_DOTENV_LOCATION=/usr/cloud/portal/.env
+  # Install dependencies
+RUN pipenv install -d
+# Copy sources
+COPY ./recircle ./recircle
+# Set working dir where the manage.py found
+WORKDIR /usr/cloud/portal/recircle
+# Migrate and start server
+CMD pipenv run python manage.py migrate \
+    && pipenv run python manage.py runserver 0.0.0.0:8000
+

Dockerfile.prod

+FROM python:3.6
+WORKDIR /usr/cloud/portal 
+SHELL ["/bin/bash", "-c"]
+# Install pipenv
+RUN pip install pipenv
+# Open 8000 port
+EXPOSE 8000/tcp
+# Copy files necessary for install dependencies
+COPY Pipfile* .env ./
+# This env variable needed to pipenv find .env file
+ENV PIPENV_DOTENV_LOCATION=/usr/cloud/portal/.env
+# Install dependencies
+RUN pipenv install
+# Copy sources
+COPY ./recircle ./recircle
+# Set working dir where the manage.py found
+WORKDIR /usr/cloud/portal/recircle
+# Migrate and start server
+CMD pipenv run python manage.py migrate \
+    && pipenv run python manage.py collectstatic --no-input --clear \
+    && pipenv run gunicorn recircle.wsgi:application --bind 0.0.0.0:8000
+

Pipfile

@@ -6,10 +6,8 @@ verify_ssl = true
 [dev-packages]
 httpie = "*"
 flake8 = "*"
-django-rest-swagger = "*"
 coverage = "*"
 django-nose = "*"
-v = {editable = true,version = "*"}
 
 [packages]
 django = "*"
@@ -24,6 +22,9 @@ channels = "*"
 channels-redis = "*"
 celery = {extras = ["redis"],version = "*"}
 django-celery-beat = "*"
+gunicorn = "*"
+psycopg2-binary = "*"
+daphne = "*"
 
 [requires]
 python_version = "3.6"

config/nginx/Dockerfile

+FROM nginx:1.19-alpine
+
+RUN rm /etc/nginx/conf.d/default.conf
+COPY ./recircle.conf /etc/nginx/conf.d
\ No newline at end of file

config/nginx/recircle.conf

+upstream recircle {
+    server backend:8000;
+}
+
+upstream wsbackend {
+    server daphne-server:8001;
+}
+
+server {
+
+    listen 80;
+
+    location /admin {
+        proxy_pass http://recircle;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /api/v1/ {
+        proxy_pass http://recircle;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /auth/ {
+        proxy_pass http://recircle;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /ws/ {
+      proxy_pass http://wsbackend;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "Upgrade";
+      proxy_set_header Host $host;
+}
+
+    
+    location /static/ {
+        alias /static/;
+    }
+
+}
\ No newline at end of file

docker-compose.prod.yml

+version: "3.8"
+
+services:
+  nginx:
+    build: ./config/nginx
+    image: recircle-nginx
+    ports:
+      - "80:80"
+    volumes:
+      - "/static:/static"
+    networks:
+      - backend
+    depends_on:
+      - backend
+
+  db:
+    image: postgres:13-alpine
+    environment:
+      POSTGRES_PASSWORD: ${DATABASE_PASSWORD}
+      POSTGRES_USER: ${DATABASE_USER}
+      POSTGRES_DB: ${DATABASE_NAME}
+    volumes:
+      - /database/recircle:/var/lib/postgresql/data
+    networks:
+      - backend
+
+  redis:
+    image: redis:alpine
+    networks:
+      - backend
+
+  backend:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.prod
+    networks:
+      - backend
+    expose:
+      - "8000"
+    volumes:
+      - "/static:/static"
+
+    depends_on:
+      - db
+      - redis
+
+  celery:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.prod
+    command: pipenv run celery -A recircle worker -B -l INFO
+    networks:
+      - backend
+    depends_on:
+      - backend
+      - db
+      - redis
+
+  daphne-server:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.prod
+    command: pipenv run daphne -b 0.0.0.0 -p 8001 recircle.asgi:application
+    expose:
+      - "8001"
+    networks:
+      - backend
+    depends_on:
+      - backend
+      - db
+      - redis
+
+networks:
+  backend:

docker-compose.yml

+version: "3.8"
+
+services:
+  backend:
+    build: .
+    image: portal:compose
+    ports:
+      - "8000:8000"
+    networks:
+      - backend
+    depends_on:
+      - db
+      - redis
+
+  db:
+    image: postgres:13-alpine
+    environment:
+      POSTGRES_PASSWORD: ${DATABASE_PASSWORD}
+      POSTGRES_USER: ${DATABASE_USER}
+      POSTGRES_DB: ${DATABASE_NAME}
+    networks:
+      - backend
+
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379"
+    networks:
+      - backend
+
+networks:
+  backend:

environment.sh.example

-export DJANGO_SECRET = "very very secret key"
+# This file sets the environment variables in bare metal or 
+# Virtual machine deployment
+# If you prefer deploy with docker, than use the .env file instead this
+export DJANGO_SECRET="very very secret key"
+# settings.local for development and testing
+# settings.production for production deployment
+export DJANGO_SETTINGS_MODULE="myproject.settings.local"
+# If using other database than sqlite3 (only use sqlite for development!)
+# Provide the db credentials here
+export DATABASE_PASSWORD="database_password"
+export DATABASE_USER="user"
+export DATABASE_NAME="user"
+export DATABASE_HOST="hostname_of_batabase"
+# For channels communication and periodic tasks we need a redis server
+export REDIS_HOST="redis_host"
+# OpenStack credentials
+# In development we recommend using a DevStack installation
+# These settings displayed in an admin dashboard or
+# Can be downloaded from Horizon, as an admin user
 export OS_AUTH_URL=http://example/identity/v3
 export OS_PROJECT_ID=123456789
 export OS_PROJECT_NAME="example"

recircle/recircle/asgi.py

+"""
+ASGI entrypoint. Configures Django and then runs the application
+defined in the ASGI_APPLICATION setting.
+"""
+
+import os
+import django
+from channels.routing import get_default_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "myproject.settings.base")
+django.setup()
+application = get_default_application()

recircle/recircle/celery.py

@@ -13,9 +13,7 @@ app = Celery('recircle')
 # the configuration object to child processes.
 # - namespace='CELERY' means all celery-related configuration keys
 #   should have a `CELERY_` prefix.
-# app.config_from_object('django.conf:settings', namespace='CELERY')
-
-app.conf.broker_url = 'redis://localhost:6379/0'
+app.config_from_object('django.conf:settings', namespace='CELERY')
 
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()

recircle/recircle/settings/base.py

@@ -20,7 +20,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = "xhe8=w50lz82gjb6+z%(rwk2c+1kd!%(iv_s^!tp)*5cnb=-^t"
+SECRET_KEY = os.getenv("DJANGO_SECRET")
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
@@ -39,10 +39,8 @@ INSTALLED_APPS = [
     "rest_framework",
     "rest_framework.authtoken",
     "djoser",
-    "rest_framework_swagger",
     "corsheaders",
     "guardian",
-    "django_nose",
     "channels",
     "django_celery_beat"
 ]
@@ -253,4 +251,8 @@ CHANNEL_LAYERS = {
     },
 }
 
+CELERY_BROKER_URL = 'redis://localhost:6379'
+CELERY_RESULT_BACKEND = "redis://localhost:6379"
+
+
 ASGI_APPLICATION = "recircle.routing.application"

recircle/recircle/settings/local.py

@@ -10,6 +10,10 @@ for i in LOCAL_APPS:
 
 ADMIN_ENABLED = True
 
+INSTALLED_APPS += [
+    'django_nose'
+]
+
 ALLOWED_HOSTS = ['*']
 
 AUTH_PASSWORD_VALIDATORS = []

recircle/recircle/settings/production.py

+import os
+from .base import *
+
+DEBUG = False
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "HOST": os.getenv("DATABASE_HOST"),
+        "PORT": "5432",
+        "NAME": os.getenv("DATABASE_NAME"),
+        "USER": os.getenv("DATABASE_USER"),
+        "PASSWORD": os.getenv("DATABASE_PASSWORD"),
+       }
+}
+
+STATIC_ROOT = "/static/"
+
+CELERY_BROKER_URL = 'redis://redis:6379'
+CELERY_RESULT_BACKEND = "redis://redis:6379"
+
+CHANNEL_LAYERS = {
+    'default': {
+        'BACKEND': 'channels_redis.core.RedisChannelLayer',
+        'CONFIG': {
+            "hosts": [(os.getenv('REDIS_HOST', 'redis'), 6379)],
+        },
+    },
+}
\ No newline at end of file

recircle/recircle/urls.py

@@ -15,10 +15,6 @@ Including another URLconf
 """
 from django.contrib import admin
 from django.urls import path, re_path, include
-from rest_framework_swagger.views import get_swagger_view
-
-
-schema_view = get_swagger_view(title="RECIRCLE API")
 
 urlpatterns = [
     path("api/v1/", include("image.urls")),
@@ -27,5 +23,4 @@ urlpatterns = [
     path("admin/", admin.site.urls),
     re_path(r"^auth/", include("djoser.urls")),
     re_path(r"^auth/", include("djoser.urls.authtoken")),
-    path(r"swagger", schema_view),
 ]

setup_ci_env.sh

+#!/bin/bash
+## This file used by the gitlab ci runner
+## Reads the environment variables from the CI config then echoes it to the .env file
+# This variables are interpreted automatically by the Docker Engine
+# And by the Django runtime
+# Django configuration
+echo DJANGO_SECRET=${DJANGO_SECRET} >> .env
+echo DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} >> .env
+# If using other database than sqlite3 (only use sqlite for development!)
+# Provide the db credentials here
+echo DATABASE_PASSWORD=${DATABASE_PASSWORD} >> .env
+echo DATABASE_USER=${DATABASE_USER} >> .env
+echo DATABASE_NAME=${DATABASE_NAME} >> .env
+echo DATABASE_HOST=${DATABASE_HOST} >> .env
+# For channels communication and periodic tasks we need a redis server
+echo REDIS_HOST=${REDIS_HOST} >> .env
+# OpenStack credentials
+# In development we recommend using a DevStack installation
+# These settings displayed in an admin dashboard or
+# Can be downloaded from Horizon, as an admin user
+echo OS_AUTH_URL=${OS_AUTH_URL} >> .env
+echo OS_PROJECT_ID=${OS_PROJECT_ID} >> .env
+echo OS_PROJECT_NAME=${OS_PROJECT_NAME} >> .env
+echo OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME} >> .env
+echo OS_USER_DOMAIN_ID=${OS_USER_DOMAIN_ID} >> .env
+echo OS_PROJECT_DOMAIN_ID=${OS_PROJECT_DOMAIN_ID} >> .env
+echo OS_USERNAME=${OS_USERNAME} >> .env
+echo OS_PASSWORD=${OS_PASSWORD} >> .env
+echo OS_REGION_NAME=${OS_REGION_NAME} >> .env
+echo OS_INTERFACE=${OS_INTERFACE} >> .env
+echo OS_IDENTITY_API_VERSION=${OS_IDENTITY_API_VERSION} >> .env
\ No newline at end of file