Commit 9da543ec authored by Nikos Pappas's avatar Nikos Pappas
Browse files

3rd

parent a04ac112
DEBUG=True
DJANGO_WARN_LEVEL=DEBUG
DJANGO_SECRET_KEY=CHANGEME
DJANGO_SECRET_KEY=
DJANGO_TIME_ZONE='Europe/Athens'
EMD_EL_BASE_URL='https://reg-diavlos.gov.gr/'
EMD_SCHEME=https
EMD_PATH=/
EMD_EL_BASE_URL=reg-diavlos.gov.gr
EMD_EL_USERNAME=Api-bot
EMD_EL_PASSWORD=
EMD_EN_BASE_URL=en.reg-diavlos.gov.gr
EMD_EN_USERNAME=Api-bot
EMD_EN_PASSWORD=
......@@ -22,4 +22,14 @@ After installing, run:
1. `python manage.py makemigrations && python manage.py migrate`
2. `python manage.py createsuperuser`: Enter credentials for a superuser.
3. `python manage.py runserver`
4. Enter `http://localhost:8000/admin` and login with the credentials you gave.
\ No newline at end of file
4. Enter `http://localhost:8000/admin` and login with the credentials you gave.
### Api endpoints
- api/evidences : GET, POST, PUT, DELETE
- users/ : Same
With debug mode on:
- docs/swagger-ui/ : Swagger ui
- docs/schema/redoc/ : Another ui
- docs/schema/ : Schema download
\ No newline at end of file
......@@ -4,9 +4,9 @@ from django.utils.translation import ugettext_lazy as _
class Evidence(models.Model):
title = models.TextField(help_text=_('Evidence title'))
mw_uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, help_text=_('Evidence uuid'))
mw_id = models.CharField(max_length=255, help_text=_('Evidence id'))
title = models.TextField(help_text=_('Evidence title'), unique=True)
mw_uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True, help_text=_('Evidence uuid'))
mw_id = models.CharField(max_length=255, help_text=_('Evidence id'), unique=True)
class Meta:
verbose_name = 'Δικαιολογητικό'
......
......@@ -5,4 +5,4 @@ from .models import Evidence
class EvidenceSerializer(serializers.ModelSerializer):
class Meta:
model = Evidence
fields = ('title', 'mw_uuid', 'mw_id')
\ No newline at end of file
fields = ('title', 'mw_uuid', 'mw_id')
......@@ -2,12 +2,14 @@ from django.contrib import admin
from rest_framework import viewsets
from rest_framework import status
from rest_framework.response import Response
from rest_framework.decorators import api_view
from rest_framework.permissions import IsAuthenticatedOrReadOnly, DjangoModelPermissionsOrAnonReadOnly
# App specific imports
from .models import Evidence
from .serializers import EvidenceSerializer
from utils import get_evidences_from_mw
from utils.MediawikiConnector import Site
admin.autodiscover()
......@@ -18,10 +20,16 @@ class EvidenceViewSet(viewsets.ModelViewSet):
"""
queryset = Evidence.objects.all().order_by('title')
serializer_class = EvidenceSerializer
permission_classes = [IsAuthenticatedOrReadOnly] # Only allow authenticated users to POST
@api_view(('GET',))
def get_evidences_from_emd(request):
evidences_json = get_evidences_from_mw.fetch_evidences()
lang = request.GET.get('lang', 'gr') # set language for emd
site = Site(lang=lang)
evidences_json = site.get_evidences_from_emd()
if evidences_json is None:
return Response({}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return Response(evidences_json, status=status.HTTP_200_OK)
#!/usr/bin/env bash
# Installs
dpkg -s "python3-pip" &> /dev/null
if [ $? -nq 0 ]; then
sudo -u ${SUDO_USER} apt-get install -y python3-pip
python=$(which python3-pip)
echo "$python"
if [ -z "$python" ]; then
sudo apt-get install -y python3-pip
fi
......
......@@ -158,10 +158,14 @@ LOGGING = {
REST_FRAMEWORK = {
'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', # schema
'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
'rest_framework.permissions.IsAuthenticatedOrReadOnly'
],
"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
"PAGE_SIZE": 50,
# 'DEFAULT_AUTHENTICATION_CLASSES': (
# 'rest_framework.authentication.BasicAuthentication',
# 'rest_framework.authentication.SessionAuthentication',
# ),
}
# Swagger settings
......
......@@ -20,5 +20,6 @@ setup(
"drf-spectacular",
"requests",
"tablib",
"mwclient @ git+https://github.com/mwclient/mwclient#egg=mwclient"
],
)
......@@ -16,6 +16,10 @@ body {
color: var(--body-fg)!important;
}
h2 {
font-size: 16px!important;
}
/*
Admin
*/
......
......@@ -30,9 +30,8 @@ $( document ).ready(function (){
$(target).css('opacity', '0.8');
spinner.spin(target);
$.get('http://localhost:8000/get_evidences/').done( (data) => {
console.log(JSON.parse(data));
// const str = JSON.stringify({sk: "sld"});
$.get('http://localhost:8000/api/get_evidences_from_emd/').done( (data) => {
// console.log(JSON.parse(data));
const bytes = new TextEncoder().encode(data);
const blob = new Blob([bytes], {
type: "application/json;charset=utf-8"
......
from rest_framework import serializers
from .models import User
from django.contrib.auth.models import Group
from django.core import exceptions
import django.contrib.auth.password_validation as validators
class GroupSerializer(serializers.ModelSerializer):
......@@ -10,12 +12,51 @@ class GroupSerializer(serializers.ModelSerializer):
class UserSerializer(serializers.ModelSerializer):
groups = GroupSerializer(many=True)
groups = GroupSerializer(many=True, required=False)
model = User
class Meta:
model = User
fields = ('id', 'username', 'first_name', 'last_name', 'afm', 'email', 'from_gsis', 'groups',)
fields = ('id', 'username', 'password', 'first_name', 'last_name', 'afm', 'email', 'from_gsis', 'groups',)
extra_kwargs = {
'password': {'write_only': True},
'id': {'read_only': True}
}
def validate(self, data):
# here data has all the fields which have validated values
# so we can create a User instance out of it
user = User(**data)
# get the password from the data
password = data.get('password')
errors = dict()
try:
# validate the password and catch the exception
validators.validate_password(password=password, user=user)
# the exception raised here is different than serializers.ValidationError
except exceptions.ValidationError as e:
errors['password'] = list(e.messages)
if errors:
raise serializers.ValidationError(errors)
return super(UserSerializer, self).validate(data)
def create(self, validated_data):
user = User.objects.create(
username=validated_data['username'],
email=validated_data['email'],
first_name=validated_data['first_name'],
last_name=validated_data['last_name']
)
user.set_password(validated_data['password'])
user.save()
return user
from rest_framework import viewsets
from rest_framework import permissions
from rest_framework import authentication
from .serializers import UserSerializer
from .models import User
......@@ -11,4 +12,4 @@ class UserViewSet(viewsets.ModelViewSet):
"""
serializer_class = UserSerializer
queryset = User.objects.all().order_by('id')
permission_classes = [permissions.IsAuthenticated]
\ No newline at end of file
permission_classes = [permissions.IsAuthenticated]
"""
A module for storing configuration for accessing mediawiki site
Default values are provided, but passwords must be changed in docker-compose file
"""
import logging
import os
import environ
from nationalRegistry.settings import env
logger = logging.getLogger(__name__)
class MediaWikiConfig:
_DEFAULT_SCHEME = 'https'
_DEFAULT_PATH = '/'
_DEFAULT_URL = 'reg-diavlos.gov.gr'
env = environ.Env(
EMD_SCHEME=(str, 'https')
)
def __init__(self, lang='gr'):
self.lang = lang
# getting configuration from .env
self.scheme = env.str('EMD_SCHEME')
self.path = env.str('EMD_PATH')
if lang == 'gr' or lang == 'GR':
self.url = env.str('EMD_EL_BASE_URL')
self.username = env.str('EMD_EL_USERNAME')
self.password = env.str('EMD_EL_PASSWORD')
else:
self.url = env.str('EMD_EN_BASE_URL')
self.username = env.str('EMD_EN_USERNAME')
self.password = os.getenv('EMD_EN_PASSWORD', 'CHANGEME')
# self.password = env('EMD_EN_PASSWORD')
def __str__(self):
return self.lang
import mwclient
import tablib
import logging
from .MediaWikiConfig import MediaWikiConfig
from .helper_functions import parse_processes
logger = logging.getLogger(__name__)
class Site:
_DEFAULT_LANG = 'gr'
_DEFAULT_SCHEME = 'https'
_DEFAULT_PATH = '/'
def __init__(self, lang=_DEFAULT_LANG):
self.lang = lang
self.__client = None
self.__config = None
self._logged_in = False
self.api = self._client.api
self.get = self._client.get
@property
def _client(self):
if self.__client is None:
client = mwclient.Site(
self._config.url,
scheme=self._config.scheme,
path=self._config.path
)
self.__client = client
return self.__client
@property
def _config(self):
if self.__config is None:
self.__config = MediaWikiConfig(self.lang)
return self.__config
def login(self, username='', password='', auto=False, force=False):
"""Login by username and password."""
if self._logged_in and not force:
return
if auto:
username = self._config.username
password = self._config.password.lstrip('l')
try:
self._client.clientlogin(username=username, password=password)
except (mwclient.errors.LoginError,
mwclient.errors.APIError) as e:
logger.error(e)
else:
self._logged_in = True
def _call_to_api(self):
english = True if self.lang == 'en' or self.lang == 'EN' else False
limit = 500
offset = 0
parameters = ''
processes = []
while 1:
try:
askargs_conditions = f'Category:Services Registry' if english else \
f'Category:Κατάλογος Διαδικασιών'
parameters = parameters + f'|limit={limit}' + f'|offset={offset}'
mw_response = self.get(
'askargs', format='json',
api_version=3,
conditions=askargs_conditions,
parameters=parameters,
printouts='Process evidence description'
)
except mwclient.errors.APIError:
logger.error(mwclient.errors.APIError)
exit(1)
else:
if 'query-continue-offset' in mw_response:
offset = mw_response['query-continue-offset']
processes.extend(mw_response['query']['results'])
else:
break
return processes
def get_evidences_from_emd(self):
self.login(auto=True)
processes = self._call_to_api()
emd_evidences_array = parse_processes(processes)
exported_data = tablib.Dataset()
exported_data.headers = ['id', 'title', 'mw_uuid', 'mw_id']
index = 0
for i in emd_evidences_array:
exported_data.append([index, i, '', ''])
index = index + 1
return exported_data.json
import tablib
import requests
import environ
import logging
from requests.exceptions import RequestException
from nationalRegistry.settings import env
logger = logging.getLogger(__name__)
# Load .env
# env = environ.Env.read_env()
query = '/api.php?action=ask&query=%5B%5B%CE%9A%CE%B1%CF%84%CE%B7%CE%B3%CE%BF%CF%81%CE%AF%CE%B1%3A%CE%9A%CE%B1%CF%84%CE%AC%CE%BB%CE%BF%CE%B3%CE%BF%CF%82+%CE%94%CE%B9%CE%B1%CE%B4%CE%B9%CE%BA%CE%B1%CF%83%CE%B9%CF%8E%CE%BD%5D%5D%0A%5B%5BProcess+evidence+description%3A%3A%2B%5D%5D|?Process%20evidence%20description'
def get_processes_from_emd():
"""
Multiple Api calls to emd api to fetch the processes that contain Property:Process evidence description
Returns:
dict: Dictionary of evidences
Raises:
requests.exceptions.RequestException:
"""
base_url = 'https://reg-diavlos.gov.gr/api.php?action=ask&query=%5B%5B%CE%9A%CE%B1%CF%84%CE%B7%CE%B3%CE%BF%CF%81%CE%AF%CE%B1%3A%CE%9A%CE%B1%CF%84%CE%AC%CE%BB%CE%BF%CE%B3%CE%BF%CF%82+%CE%94%CE%B9%CE%B1%CE%B4%CE%B9%CE%BA%CE%B1%CF%83%CE%B9%CF%8E%CE%BD%5D%5D%0A%5B%5BProcess+evidence+description%3A%3A%2B%5D%5D|?Process%20evidence%20description'
url = base_url + query
limit = 500
offset = 0
processes = dict()
while 1:
try:
url_with_params = url + f'|limit={limit}|offset={offset}&format=json'
r = requests.get(url_with_params)
print(r.status_code)
data = r.json()
offset = data['query-continue-offset']
print(url_with_params)
except RequestException as e:
logger.error(e)
return None
except AssertionError:
print('skata', flush=True)
break
except KeyError:
break
else:
processes.update(data['query']['results'])
return processes
def parse_processes(processes):
"""Parse dict processes and transform to array
Args:
processes: The processes fetched from National Procedures Registry
Returns:
emd_evidences_array: The processes as array
Raises:
"""
emd_evidences_array = []
for process in processes.values():
curr_evidences = process['printouts']['Process evidence description']
# Remove trailing dots
# Remove spaces at the beginning and at the end of the string
formatted_evidences = [s.strip().rstrip('.').strip('\"') for s in curr_evidences]
emd_evidences_array += formatted_evidences
emd_evidences_array.sort()
# remove duplicates
emd_evidences_array = list(set(emd_evidences_array))
return emd_evidences_array
def fetch_evidences():
"""Fetch the evidences from national procedures registry
Returns:
json: The evidences as json to import to django-import-export app
"""
processes = get_processes_from_emd()
if processes is None:
return None
emd_evidences_array = parse_processes(processes)
exported_data = tablib.Dataset()
exported_data.headers = ['id', 'mw_title', 'mw_uuid', 'mw_id']
index = 0
for i in emd_evidences_array:
exported_data.append([index, i, '', ''])
index = index + 1
return exported_data.json
\ No newline at end of file
import logging
logger = logging.getLogger(__name__)
def parse_processes(processes):
"""Parse dict processes and transform to array
Args:
processes: The processes fetched from National Procedures Registry
Returns:
emd_output_array: The processes as array
Raises:
"""
output_array = []
for process in processes:
curr_evidences = list(list(process.values())[0]['printouts'].values())[0]
if len(curr_evidences) != 0:
# Remove trailing dots
# Remove spaces at the beginning and at the end of the string
formatted_evidences = [s.strip().rstrip('.').strip('\"') for s in curr_evidences]
output_array += formatted_evidences
# sort
output_array.sort()
# remove duplicates
emd_output_array = list(set(output_array))
return emd_output_array
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment