3rd

.env.sample

 DEBUG=True
 DJANGO_WARN_LEVEL=DEBUG
-DJANGO_SECRET_KEY=CHANGEME
+DJANGO_SECRET_KEY=
 DJANGO_TIME_ZONE='Europe/Athens'
 
-EMD_EL_BASE_URL='https://reg-diavlos.gov.gr/'
+EMD_SCHEME=https
+EMD_PATH=/
+EMD_EL_BASE_URL=reg-diavlos.gov.gr
+EMD_EL_USERNAME=Api-bot
+EMD_EL_PASSWORD=
+EMD_EN_BASE_URL=en.reg-diavlos.gov.gr
+EMD_EN_USERNAME=Api-bot
+EMD_EN_PASSWORD=

README.md

@@ -22,4 +22,14 @@ After installing, run:
 1. `python manage.py makemigrations && python manage.py migrate`
 2. `python manage.py createsuperuser`: Enter credentials for a superuser.
 3. `python manage.py runserver`
-4. Enter `http://localhost:8000/admin` and login with the credentials you gave.
\ No newline at end of file
+4. Enter `http://localhost:8000/admin` and login with the credentials you gave.
+
+
+### Api endpoints
+- api/evidences : GET, POST, PUT, DELETE
+- users/ : Same
+
+With debug mode on:
+- docs/swagger-ui/ : Swagger ui
+- docs/schema/redoc/ : Another ui
+- docs/schema/ : Schema download
\ No newline at end of file

evidences/models.py

@@ -4,9 +4,9 @@ from django.utils.translation import ugettext_lazy as _
 
 
 class Evidence(models.Model):
-    title = models.TextField(help_text=_('Evidence title'))
-    mw_uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, help_text=_('Evidence uuid'))
-    mw_id = models.CharField(max_length=255, help_text=_('Evidence id'))
+    title = models.TextField(help_text=_('Evidence title'), unique=True)
+    mw_uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True, help_text=_('Evidence uuid'))
+    mw_id = models.CharField(max_length=255, help_text=_('Evidence id'), unique=True)
 
     class Meta:
         verbose_name = 'Δικαιολογητικό'

evidences/serializers.py

@@ -5,4 +5,4 @@ from .models import Evidence
 class EvidenceSerializer(serializers.ModelSerializer):
     class Meta:
         model = Evidence
-        fields = ('title', 'mw_uuid', 'mw_id')
\ No newline at end of file
+        fields = ('title', 'mw_uuid', 'mw_id')

evidences/views.py

@@ -2,12 +2,14 @@ from django.contrib import admin
 from rest_framework import viewsets
 from rest_framework import status
 from rest_framework.response import Response
+from rest_framework.decorators import api_view
+from rest_framework.permissions import IsAuthenticatedOrReadOnly, DjangoModelPermissionsOrAnonReadOnly
 
 # App specific imports
 from .models import Evidence
 from .serializers import EvidenceSerializer
 
-from utils import get_evidences_from_mw
+from utils.MediawikiConnector import Site
 
 admin.autodiscover()
 
@@ -18,10 +20,16 @@ class EvidenceViewSet(viewsets.ModelViewSet):
     """
     queryset = Evidence.objects.all().order_by('title')
     serializer_class = EvidenceSerializer
+    permission_classes = [IsAuthenticatedOrReadOnly]  # Only allow authenticated users to POST
 
 
+@api_view(('GET',))
 def get_evidences_from_emd(request):
-    evidences_json = get_evidences_from_mw.fetch_evidences()
+    lang = request.GET.get('lang', 'gr')  # set language for emd
+
+    site = Site(lang=lang)
+
+    evidences_json = site.get_evidences_from_emd()
     if evidences_json is None:
         return Response({}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
     return Response(evidences_json, status=status.HTTP_200_OK)

install.sh

 #!/usr/bin/env bash
 
 # Installs
-
 dpkg -s "python3-pip" &> /dev/null
-if [ $? -nq 0 ]; then
-  sudo -u ${SUDO_USER} apt-get install -y python3-pip
+python=$(which python3-pip)
+echo "$python"
+
+if [ -z "$python" ]; then
+  sudo apt-get install -y python3-pip
 fi
 
 

nationalRegistry/settings.py

@@ -158,10 +158,14 @@ LOGGING = {
 REST_FRAMEWORK = {
     'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',  # schema
     'DEFAULT_PERMISSION_CLASSES': [
-        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
+        'rest_framework.permissions.IsAuthenticatedOrReadOnly'
     ],
     "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
     "PAGE_SIZE": 50,
+    # 'DEFAULT_AUTHENTICATION_CLASSES': (
+    #     'rest_framework.authentication.BasicAuthentication',
+    #     'rest_framework.authentication.SessionAuthentication',
+    # ),
 }
 
 # Swagger settings

setup.py

@@ -20,5 +20,6 @@ setup(
         "drf-spectacular",
         "requests",
         "tablib",
+        "mwclient @ git+https://github.com/mwclient/mwclient#egg=mwclient"
     ],
 )

static/css/custom/custom.css

@@ -16,6 +16,10 @@ body {
     color: var(--body-fg)!important;
 }
 
+h2 {
+    font-size: 16px!important;
+}
+
 /*
 Admin
 */

static/js/custom/admin-import.js

@@ -30,9 +30,8 @@ $( document ).ready(function (){
         $(target).css('opacity', '0.8');
         spinner.spin(target);
 
-        $.get('http://localhost:8000/get_evidences/').done( (data) => {
-            console.log(JSON.parse(data));
-            // const str = JSON.stringify({sk: "sld"});
+        $.get('http://localhost:8000/api/get_evidences_from_emd/').done( (data) => {
+            // console.log(JSON.parse(data));
             const bytes = new TextEncoder().encode(data);
             const blob = new Blob([bytes], {
                 type: "application/json;charset=utf-8"

users/serializers.py

 from rest_framework import serializers
 from .models import User
 from django.contrib.auth.models import Group
+from django.core import exceptions
+import django.contrib.auth.password_validation as validators
 
 
 class GroupSerializer(serializers.ModelSerializer):
@@ -10,12 +12,51 @@ class GroupSerializer(serializers.ModelSerializer):
 
 
 class UserSerializer(serializers.ModelSerializer):
-    groups = GroupSerializer(many=True)
+    groups = GroupSerializer(many=True, required=False)
     model = User
 
     class Meta:
         model = User
-        fields = ('id', 'username', 'first_name', 'last_name', 'afm', 'email', 'from_gsis', 'groups',)
+        fields = ('id', 'username', 'password', 'first_name', 'last_name', 'afm', 'email', 'from_gsis', 'groups',)
+        extra_kwargs = {
+            'password': {'write_only': True},
+            'id': {'read_only': True}
+        }
+
+    def validate(self, data):
+        # here data has all the fields which have validated values
+        # so we can create a User instance out of it
+        user = User(**data)
+
+        # get the password from the data
+        password = data.get('password')
+
+        errors = dict()
+        try:
+            # validate the password and catch the exception
+            validators.validate_password(password=password, user=user)
+
+        # the exception raised here is different than serializers.ValidationError
+        except exceptions.ValidationError as e:
+            errors['password'] = list(e.messages)
+
+        if errors:
+            raise serializers.ValidationError(errors)
+
+        return super(UserSerializer, self).validate(data)
+
+    def create(self, validated_data):
+        user = User.objects.create(
+            username=validated_data['username'],
+            email=validated_data['email'],
+            first_name=validated_data['first_name'],
+            last_name=validated_data['last_name']
+        )
+
+        user.set_password(validated_data['password'])
+        user.save()
+
+        return user
 
 
 

users/views.py

 from rest_framework import viewsets
 from rest_framework import permissions
+from rest_framework import authentication
 
 from .serializers import UserSerializer
 from .models import User
@@ -11,4 +12,4 @@ class UserViewSet(viewsets.ModelViewSet):
     """
     serializer_class = UserSerializer
     queryset = User.objects.all().order_by('id')
-    permission_classes = [permissions.IsAuthenticated]
\ No newline at end of file
+    permission_classes = [permissions.IsAuthenticated]

utils/MediaWikiConfig.py

+"""
+A module for storing configuration for accessing mediawiki site
+Default values are provided, but passwords must be changed in docker-compose file
+"""
+import logging
+import os
+
+import environ
+from nationalRegistry.settings import env
+
+logger = logging.getLogger(__name__)
+
+
+class MediaWikiConfig:
+    _DEFAULT_SCHEME = 'https'
+    _DEFAULT_PATH = '/'
+    _DEFAULT_URL = 'reg-diavlos.gov.gr'
+    env = environ.Env(
+        EMD_SCHEME=(str, 'https')
+    )
+
+    def __init__(self, lang='gr'):
+        self.lang = lang
+
+        # getting configuration from .env
+        self.scheme = env.str('EMD_SCHEME')
+        self.path = env.str('EMD_PATH')
+
+        if lang == 'gr' or lang == 'GR':
+            self.url = env.str('EMD_EL_BASE_URL')
+            self.username = env.str('EMD_EL_USERNAME')
+            self.password = env.str('EMD_EL_PASSWORD')
+        else:
+            self.url = env.str('EMD_EN_BASE_URL')
+            self.username = env.str('EMD_EN_USERNAME')
+            self.password = os.getenv('EMD_EN_PASSWORD', 'CHANGEME')
+            # self.password = env('EMD_EN_PASSWORD')
+
+    def __str__(self):
+        return self.lang

utils/MediawikiConnector.py

+import mwclient
+import tablib
+import logging
+from .MediaWikiConfig import MediaWikiConfig
+from .helper_functions import parse_processes
+
+logger = logging.getLogger(__name__)
+
+
+class Site:
+    _DEFAULT_LANG = 'gr'
+    _DEFAULT_SCHEME = 'https'
+    _DEFAULT_PATH = '/'
+
+    def __init__(self, lang=_DEFAULT_LANG):
+        self.lang = lang
+        self.__client = None
+        self.__config = None
+        self._logged_in = False
+        self.api = self._client.api
+        self.get = self._client.get
+
+    @property
+    def _client(self):
+        if self.__client is None:
+            client = mwclient.Site(
+                self._config.url,
+                scheme=self._config.scheme,
+                path=self._config.path
+            )
+
+            self.__client = client
+        return self.__client
+
+    @property
+    def _config(self):
+        if self.__config is None:
+            self.__config = MediaWikiConfig(self.lang)
+        return self.__config
+
+    def login(self, username='', password='', auto=False, force=False):
+        """Login by username and password."""
+        if self._logged_in and not force:
+            return
+        if auto:
+            username = self._config.username
+            password = self._config.password.lstrip('l')
+        try:
+            self._client.clientlogin(username=username, password=password)
+        except (mwclient.errors.LoginError,
+                mwclient.errors.APIError) as e:
+            logger.error(e)
+        else:
+            self._logged_in = True
+
+    def _call_to_api(self):
+        english = True if self.lang == 'en' or self.lang == 'EN' else False
+        limit = 500
+        offset = 0
+        parameters = ''
+
+        processes = []
+
+        while 1:
+            try:
+                askargs_conditions = f'Category:Services Registry' if english else \
+                    f'Category:Κατάλογος Διαδικασιών'
+                parameters = parameters + f'|limit={limit}' + f'|offset={offset}'
+                mw_response = self.get(
+                    'askargs', format='json',
+                    api_version=3,
+                    conditions=askargs_conditions,
+                    parameters=parameters,
+                    printouts='Process evidence description'
+                )
+            except mwclient.errors.APIError:
+                logger.error(mwclient.errors.APIError)
+                exit(1)
+            else:
+                if 'query-continue-offset' in mw_response:
+                    offset = mw_response['query-continue-offset']
+                    processes.extend(mw_response['query']['results'])
+                else:
+                    break
+
+        return processes
+
+    def get_evidences_from_emd(self):
+        self.login(auto=True)
+
+        processes = self._call_to_api()
+        emd_evidences_array = parse_processes(processes)
+
+        exported_data = tablib.Dataset()
+        exported_data.headers = ['id', 'title', 'mw_uuid', 'mw_id']
+
+        index = 0
+        for i in emd_evidences_array:
+            exported_data.append([index, i, '', ''])
+            index = index + 1
+
+        return exported_data.json

utils/get_evidences_from_mw.py

-import tablib
-import requests
-import environ
-import logging
-from requests.exceptions import RequestException
-from nationalRegistry.settings import env
-
-logger = logging.getLogger(__name__)
-
-# Load .env
-# env = environ.Env.read_env()
-
-query = '/api.php?action=ask&query=%5B%5B%CE%9A%CE%B1%CF%84%CE%B7%CE%B3%CE%BF%CF%81%CE%AF%CE%B1%3A%CE%9A%CE%B1%CF%84%CE%AC%CE%BB%CE%BF%CE%B3%CE%BF%CF%82+%CE%94%CE%B9%CE%B1%CE%B4%CE%B9%CE%BA%CE%B1%CF%83%CE%B9%CF%8E%CE%BD%5D%5D%0A%5B%5BProcess+evidence+description%3A%3A%2B%5D%5D|?Process%20evidence%20description'
-
-
-def get_processes_from_emd():
-    """
-    Multiple Api calls to emd api to fetch the processes that contain Property:Process evidence description
-    Returns:
-        dict: Dictionary of evidences
-    Raises:
-        requests.exceptions.RequestException:
-    """
-    base_url = 'https://reg-diavlos.gov.gr/api.php?action=ask&query=%5B%5B%CE%9A%CE%B1%CF%84%CE%B7%CE%B3%CE%BF%CF%81%CE%AF%CE%B1%3A%CE%9A%CE%B1%CF%84%CE%AC%CE%BB%CE%BF%CE%B3%CE%BF%CF%82+%CE%94%CE%B9%CE%B1%CE%B4%CE%B9%CE%BA%CE%B1%CF%83%CE%B9%CF%8E%CE%BD%5D%5D%0A%5B%5BProcess+evidence+description%3A%3A%2B%5D%5D|?Process%20evidence%20description'
-
-    url = base_url + query
-
-    limit = 500
-    offset = 0
-
-    processes = dict()
-
-    while 1:
-        try:
-            url_with_params = url + f'|limit={limit}|offset={offset}&format=json'
-            r = requests.get(url_with_params)
-            print(r.status_code)
-            data = r.json()
-            offset = data['query-continue-offset']
-            print(url_with_params)
-        except RequestException as e:
-            logger.error(e)
-            return None
-        except AssertionError:
-            print('skata', flush=True)
-            break
-        except KeyError:
-            break
-        else:
-            processes.update(data['query']['results'])
-
-    return processes
-
-
-def parse_processes(processes):
-    """Parse dict processes and transform to array
-    Args:
-        processes: The processes fetched from National Procedures Registry
-    Returns:
-        emd_evidences_array: The processes as array
-    Raises:
-
-    """
-
-    emd_evidences_array = []
-
-    for process in processes.values():
-        curr_evidences = process['printouts']['Process evidence description']
-        # Remove trailing dots
-        # Remove spaces at the beginning and at the end of the string
-        formatted_evidences = [s.strip().rstrip('.').strip('\"') for s in curr_evidences]
-        emd_evidences_array += formatted_evidences
-
-    emd_evidences_array.sort()
-    # remove duplicates
-    emd_evidences_array = list(set(emd_evidences_array))
-
-    return emd_evidences_array
-
-
-def fetch_evidences():
-    """Fetch the evidences from national procedures registry
-
-    Returns:
-        json: The evidences as json to import to django-import-export app
-    """
-
-    processes = get_processes_from_emd()
-    if processes is None:
-        return None
-
-    emd_evidences_array = parse_processes(processes)
-
-    exported_data = tablib.Dataset()
-    exported_data.headers = ['id', 'mw_title', 'mw_uuid', 'mw_id']
-
-    index = 0
-    for i in emd_evidences_array:
-        exported_data.append([index, i, '', ''])
-        index = index + 1
-
-    return exported_data.json
\ No newline at end of file

utils/helper_functions.py

+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def parse_processes(processes):
+    """Parse dict processes and transform to array
+    Args:
+        processes: The processes fetched from National Procedures Registry
+    Returns:
+        emd_output_array: The processes as array
+    Raises:
+
+    """
+
+    output_array = []
+
+    for process in processes:
+        curr_evidences = list(list(process.values())[0]['printouts'].values())[0]
+
+        if len(curr_evidences) != 0:
+            # Remove trailing dots
+            # Remove spaces at the beginning and at the end of the string
+            formatted_evidences = [s.strip().rstrip('.').strip('\"') for s in curr_evidences]
+            output_array += formatted_evidences
+
+    # sort
+    output_array.sort()
+    # remove duplicates
+    emd_output_array = list(set(output_array))
+
+    return emd_output_array
+
+
+
+
+