From 47bffd0654ad6a17481a1ab8debea9a9ac814891 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 1 Jun 2020 14:10:14 -0400 Subject: [PATCH 01/12] Update docker-compose to include unpp Update base image used to version 2.14 --- docker-compose.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ec398af63..dfcad7183 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ x-datamart-defaults: &datamart_defaults DATABASE_URL: postgis://postgres:@db:5432/etools_datamart DATABASE_URL_ETOOLS: postgis://postgres:@db-etools:5432/etools DATABASE_URL_PRP: postgis://postgres:@db-prp:5432/prp + DATABASE_URL_UNPP: postgis://postgres:@db-unpp:5432/unpp AUTOCREATE_USERS: "admin,123" CACHE_URL: "redis://redis:6379/1" CACHE_URL_LOCK: "redis://redis:6379/1" @@ -74,7 +75,7 @@ services: volumes: - "$PWD/build/db:/var/lib/postgresql/data" - # Rely on etools and prp database instances running locally + # Rely on etools, prp, and unpp database instances running locally db-etools: image: mdillon/postgis:9.6 container_name: datamart_etools @@ -97,6 +98,17 @@ services: volumes: - "$PWD/build/prp:/var/lib/postgresql/data" + db-unpp: + image: mdillon/postgis:9.6 + shm_size: '1gb' + container_name: datamart_unpp + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: + POSTGRES_DB: unpp + volumes: + - "$PWD/build/unpp:/var/lib/postgresql/data" + redis: image: redis:alpine container_name: datamart_redis From 9f2ebcc688949b441bf9f830769854cf9a9554d7 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 1 Jun 2020 14:12:12 -0400 Subject: [PATCH 02/12] Add initial unpp source to datamart --- .../apps/sources/unpp/__init_.py | 0 .../apps/sources/unpp/admin.py | 326 ++++++ src/etools_datamart/apps/sources/unpp/api.py | 642 ++++++++++ .../apps/sources/unpp/api_urls.py | 62 + src/etools_datamart/apps/sources/unpp/apps.py | 12 + .../apps/sources/unpp/management/__init_.py | 0 .../unpp/management/commands/__init_.py | 0 .../unpp/management/commands/inspectunpp.py | 308 +++++ .../apps/sources/unpp/models.py | 1032 +++++++++++++++++ src/etools_datamart/config/settings.py | 8 +- 10 files changed, 2389 insertions(+), 1 deletion(-) create mode 100644 src/etools_datamart/apps/sources/unpp/__init_.py create mode 100644 src/etools_datamart/apps/sources/unpp/admin.py create mode 100644 src/etools_datamart/apps/sources/unpp/api.py create mode 100644 src/etools_datamart/apps/sources/unpp/api_urls.py create mode 100644 src/etools_datamart/apps/sources/unpp/apps.py create mode 100644 src/etools_datamart/apps/sources/unpp/management/__init_.py create mode 100644 src/etools_datamart/apps/sources/unpp/management/commands/__init_.py create mode 100644 src/etools_datamart/apps/sources/unpp/management/commands/inspectunpp.py create mode 100644 src/etools_datamart/apps/sources/unpp/models.py diff --git a/src/etools_datamart/apps/sources/unpp/__init_.py b/src/etools_datamart/apps/sources/unpp/__init_.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/sources/unpp/admin.py b/src/etools_datamart/apps/sources/unpp/admin.py new file mode 100644 index 000000000..a70c30cc6 --- /dev/null +++ b/src/etools_datamart/apps/sources/unpp/admin.py @@ -0,0 +1,326 @@ +from django.contrib.admin import ModelAdmin, register + +from etools_datamart.apps.core.admin_mixins import DatamartSourceModelAdmin + +from . import models + + +@register(models.AccountUser) +class AccountUserAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AccountUserGroups) +class AccountUserGroupsAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AccountUserprofile) +class AccountUserprofileAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AgencyAgency) +class AgencyAgencyAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AgencyAgencymember) +class AgencyAgencymemberAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AgencyAgencyoffice) +class AgencyAgencyofficeAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AgencyAgencyprofile) +class AgencyAgencyprofileAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AgencyOtheragency) +class AgencyOtheragencyAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.AuthGroup) +class AuthGroupAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.CommonAdminlevel1) +class CommonAdminlevel1Admin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.CommonCommonfile) +class CommonCommonfileAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.CommonPoint) +class CommonPointAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.CommonSector) +class CommonSectorAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.CommonSpecialization) +class CommonSpecializationAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.DjangoContentType) +class DjangoContentTypeAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.DjangoSite) +class DjangoSiteAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ExternalsPartnervendornumber) +class ExternalsPartnervendornumberAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ExternalsUnicefvendordata) +class ExternalsUnicefvendordataAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.NotificationNotification) +class NotificationNotificationAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.NotificationNotifieduser) +class NotificationNotifieduserAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartner) +class PartnerPartnerAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerLocationFieldOffices) +class PartnerPartnerLocationFieldOfficesAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerauditassessment) +class PartnerPartnerauditassessmentAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerauditreport) +class PartnerPartnerauditreportAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerauthorisedofficer) +class PartnerPartnerauthorisedofficerAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerbudget) +class PartnerPartnerbudgetAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnercapacityassessment) +class PartnerPartnercapacityassessmentAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnercollaborationevidence) +class PartnerPartnercollaborationevidenceAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnercollaborationpartnership) +class PartnerPartnercollaborationpartnershipAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerdirector) +class PartnerPartnerdirectorAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerexperience) +class PartnerPartnerexperienceAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerfunding) +class PartnerPartnerfundingAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnergoverningdocument) +class PartnerPartnergoverningdocumentAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerheadorganization) +class PartnerPartnerheadorganizationAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerinternalcontrol) +class PartnerPartnerinternalcontrolAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnermailingaddress) +class PartnerPartnermailingaddressAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnermandatemission) +class PartnerPartnermandatemissionAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnermember) +class PartnerPartnermemberAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerotherinfo) +class PartnerPartnerotherinfoAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerpolicyarea) +class PartnerPartnerpolicyareaAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerprofile) +class PartnerPartnerprofileAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerregistrationdocument) +class PartnerPartnerregistrationdocumentAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerreporting) +class PartnerPartnerreportingAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.PartnerPartnerreview) +class PartnerPartnerreviewAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectApplication) +class ProjectApplicationAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectApplicationLocationsProposalOfEoi) +class ProjectApplicationLocationsProposalOfEoiAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectApplicationfeedback) +class ProjectApplicationfeedbackAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectAssessment) +class ProjectAssessmentAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectClarificationrequestanswerfile) +class ProjectClarificationrequestanswerfileAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectClarificationrequestquestion) +class ProjectClarificationrequestquestionAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectEoi) +class ProjectEoiAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectEoiFocalPoints) +class ProjectEoiFocalPointsAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectEoiInvitedPartners) +class ProjectEoiInvitedPartnersAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectEoiLocations) +class ProjectEoiLocationsAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectEoiReviewers) +class ProjectEoiReviewersAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectEoiSpecializations) +class ProjectEoiSpecializationsAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectEoiattachment) +class ProjectEoiattachmentAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ProjectPin) +class ProjectPinAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ReviewPartnerflag) +class ReviewPartnerflagAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.ReviewPartnerverification) +class ReviewPartnerverificationAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.SanctionslistSanctioneditem) +class SanctionslistSanctioneditemAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.SanctionslistSanctionedname) +class SanctionslistSanctionednameAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.SanctionslistSanctionednamematch) +class SanctionslistSanctionednamematchAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + + +@register(models.SequencesSequence) +class SequencesSequenceAdmin(DatamartSourceModelAdmin, ModelAdmin): + list_filter = [] + diff --git a/src/etools_datamart/apps/sources/unpp/api.py b/src/etools_datamart/apps/sources/unpp/api.py new file mode 100644 index 000000000..3cb937e7c --- /dev/null +++ b/src/etools_datamart/apps/sources/unpp/api.py @@ -0,0 +1,642 @@ +from unicef_rest_framework.views import URFReadOnlyModelViewSet + +from etools_datamart.api.endpoints.etools import serializers +from etools_datamart.apps.sources.unpp import models + + +class AgencyAgencySerializer(serializers.ModelSerializer): + class Meta: + model = models.AgencyAgency + exclude = () + + +class AgencyAgencyViewSet(URFReadOnlyModelViewSet): + serializer_class = AgencyAgencySerializer + queryset = models.AgencyAgency.objects.all() + + +class AgencyAgencymemberSerializer(serializers.ModelSerializer): + class Meta: + model = models.AgencyAgencymember + exclude = () + + +class AgencyAgencymemberViewSet(URFReadOnlyModelViewSet): + serializer_class = AgencyAgencymemberSerializer + queryset = models.AgencyAgencymember.objects.all() + + +class AgencyAgencyofficeSerializer(serializers.ModelSerializer): + class Meta: + model = models.AgencyAgencyoffice + exclude = () + + +class AgencyAgencyofficeViewSet(URFReadOnlyModelViewSet): + serializer_class = AgencyAgencyofficeSerializer + queryset = models.AgencyAgencyoffice.objects.all() + + +class AgencyAgencyprofileSerializer(serializers.ModelSerializer): + class Meta: + model = models.AgencyAgencyprofile + exclude = () + + +class AgencyAgencyprofileViewSet(URFReadOnlyModelViewSet): + serializer_class = AgencyAgencyprofileSerializer + queryset = models.AgencyAgencyprofile.objects.all() + + +class AgencyOtheragencySerializer(serializers.ModelSerializer): + class Meta: + model = models.AgencyOtheragency + exclude = () + + +class AgencyOtheragencyViewSet(URFReadOnlyModelViewSet): + serializer_class = AgencyOtheragencySerializer + queryset = models.AgencyOtheragency.objects.all() + + +class CommonAdminlevel1Serializer(serializers.ModelSerializer): + class Meta: + model = models.CommonAdminlevel1 + exclude = () + + +class CommonAdminlevel1ViewSet(URFReadOnlyModelViewSet): + serializer_class = CommonAdminlevel1Serializer + queryset = models.CommonAdminlevel1.objects.all() + + +class CommonCommonfileSerializer(serializers.ModelSerializer): + class Meta: + model = models.CommonCommonfile + exclude = () + + +class CommonCommonfileViewSet(URFReadOnlyModelViewSet): + serializer_class = CommonCommonfileSerializer + queryset = models.CommonCommonfile.objects.all() + + +class CommonPointSerializer(serializers.ModelSerializer): + class Meta: + model = models.CommonPoint + exclude = () + + +class CommonPointViewSet(URFReadOnlyModelViewSet): + serializer_class = CommonPointSerializer + queryset = models.CommonPoint.objects.all() + + +class CommonSectorSerializer(serializers.ModelSerializer): + class Meta: + model = models.CommonSector + exclude = () + + +class CommonSectorViewSet(URFReadOnlyModelViewSet): + serializer_class = CommonSectorSerializer + queryset = models.CommonSector.objects.all() + + +class CommonSpecializationSerializer(serializers.ModelSerializer): + class Meta: + model = models.CommonSpecialization + exclude = () + + +class CommonSpecializationViewSet(URFReadOnlyModelViewSet): + serializer_class = CommonSpecializationSerializer + queryset = models.CommonSpecialization.objects.all() + + +class ExternalsPartnervendornumberSerializer(serializers.ModelSerializer): + class Meta: + model = models.ExternalsPartnervendornumber + exclude = () + + +class ExternalsPartnervendornumberViewSet(URFReadOnlyModelViewSet): + serializer_class = ExternalsPartnervendornumberSerializer + queryset = models.ExternalsPartnervendornumber.objects.all() + + +class ExternalsUnicefvendordataSerializer(serializers.ModelSerializer): + class Meta: + model = models.ExternalsUnicefvendordata + exclude = () + + +class ExternalsUnicefvendordataViewSet(URFReadOnlyModelViewSet): + serializer_class = ExternalsUnicefvendordataSerializer + queryset = models.ExternalsUnicefvendordata.objects.all() + + +class NotificationNotificationSerializer(serializers.ModelSerializer): + class Meta: + model = models.NotificationNotification + exclude = () + + +class NotificationNotificationViewSet(URFReadOnlyModelViewSet): + serializer_class = NotificationNotificationSerializer + queryset = models.NotificationNotification.objects.all() + + +class NotificationNotifieduserSerializer(serializers.ModelSerializer): + class Meta: + model = models.NotificationNotifieduser + exclude = () + + +class NotificationNotifieduserViewSet(URFReadOnlyModelViewSet): + serializer_class = NotificationNotifieduserSerializer + queryset = models.NotificationNotifieduser.objects.all() + + +class PartnerPartnerSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartner + exclude = () + + +class PartnerPartnerViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerSerializer + queryset = models.PartnerPartner.objects.all() + + +class PartnerPartnerLocationFieldOfficesSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerLocationFieldOffices + exclude = () + + +class PartnerPartnerLocationFieldOfficesViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerLocationFieldOfficesSerializer + queryset = models.PartnerPartnerLocationFieldOffices.objects.all() + + +class PartnerPartnerauditassessmentSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerauditassessment + exclude = () + + +class PartnerPartnerauditassessmentViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerauditassessmentSerializer + queryset = models.PartnerPartnerauditassessment.objects.all() + + +class PartnerPartnerauditreportSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerauditreport + exclude = () + + +class PartnerPartnerauditreportViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerauditreportSerializer + queryset = models.PartnerPartnerauditreport.objects.all() + + +class PartnerPartnerauthorisedofficerSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerauthorisedofficer + exclude = () + + +class PartnerPartnerauthorisedofficerViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerauthorisedofficerSerializer + queryset = models.PartnerPartnerauthorisedofficer.objects.all() + + +class PartnerPartnerbudgetSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerbudget + exclude = () + + +class PartnerPartnerbudgetViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerbudgetSerializer + queryset = models.PartnerPartnerbudget.objects.all() + + +class PartnerPartnercapacityassessmentSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnercapacityassessment + exclude = () + + +class PartnerPartnercapacityassessmentViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnercapacityassessmentSerializer + queryset = models.PartnerPartnercapacityassessment.objects.all() + + +class PartnerPartnercollaborationevidenceSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnercollaborationevidence + exclude = () + + +class PartnerPartnercollaborationevidenceViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnercollaborationevidenceSerializer + queryset = models.PartnerPartnercollaborationevidence.objects.all() + + +class PartnerPartnercollaborationpartnershipSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnercollaborationpartnership + exclude = () + + +class PartnerPartnercollaborationpartnershipViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnercollaborationpartnershipSerializer + queryset = models.PartnerPartnercollaborationpartnership.objects.all() + + +class PartnerPartnerdirectorSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerdirector + exclude = () + + +class PartnerPartnerdirectorViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerdirectorSerializer + queryset = models.PartnerPartnerdirector.objects.all() + + +class PartnerPartnerexperienceSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerexperience + exclude = () + + +class PartnerPartnerexperienceViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerexperienceSerializer + queryset = models.PartnerPartnerexperience.objects.all() + + +class PartnerPartnerfundingSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerfunding + exclude = () + + +class PartnerPartnerfundingViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerfundingSerializer + queryset = models.PartnerPartnerfunding.objects.all() + + +class PartnerPartnergoverningdocumentSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnergoverningdocument + exclude = () + + +class PartnerPartnergoverningdocumentViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnergoverningdocumentSerializer + queryset = models.PartnerPartnergoverningdocument.objects.all() + + +class PartnerPartnerheadorganizationSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerheadorganization + exclude = () + + +class PartnerPartnerheadorganizationViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerheadorganizationSerializer + queryset = models.PartnerPartnerheadorganization.objects.all() + + +class PartnerPartnerinternalcontrolSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerinternalcontrol + exclude = () + + +class PartnerPartnerinternalcontrolViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerinternalcontrolSerializer + queryset = models.PartnerPartnerinternalcontrol.objects.all() + + +class PartnerPartnermailingaddressSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnermailingaddress + exclude = () + + +class PartnerPartnermailingaddressViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnermailingaddressSerializer + queryset = models.PartnerPartnermailingaddress.objects.all() + + +class PartnerPartnermandatemissionSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnermandatemission + exclude = () + + +class PartnerPartnermandatemissionViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnermandatemissionSerializer + queryset = models.PartnerPartnermandatemission.objects.all() + + +class PartnerPartnermemberSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnermember + exclude = () + + +class PartnerPartnermemberViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnermemberSerializer + queryset = models.PartnerPartnermember.objects.all() + + +class PartnerPartnerotherinfoSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerotherinfo + exclude = () + + +class PartnerPartnerotherinfoViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerotherinfoSerializer + queryset = models.PartnerPartnerotherinfo.objects.all() + + +class PartnerPartnerpolicyareaSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerpolicyarea + exclude = () + + +class PartnerPartnerpolicyareaViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerpolicyareaSerializer + queryset = models.PartnerPartnerpolicyarea.objects.all() + + +class PartnerPartnerprofileSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerprofile + exclude = () + + +class PartnerPartnerprofileViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerprofileSerializer + queryset = models.PartnerPartnerprofile.objects.all() + + +class PartnerPartnerregistrationdocumentSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerregistrationdocument + exclude = () + + +class PartnerPartnerregistrationdocumentViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerregistrationdocumentSerializer + queryset = models.PartnerPartnerregistrationdocument.objects.all() + + +class PartnerPartnerreportingSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerreporting + exclude = () + + +class PartnerPartnerreportingViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerreportingSerializer + queryset = models.PartnerPartnerreporting.objects.all() + + +class PartnerPartnerreviewSerializer(serializers.ModelSerializer): + class Meta: + model = models.PartnerPartnerreview + exclude = () + + +class PartnerPartnerreviewViewSet(URFReadOnlyModelViewSet): + serializer_class = PartnerPartnerreviewSerializer + queryset = models.PartnerPartnerreview.objects.all() + + +class ProjectApplicationSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectApplication + exclude = () + + +class ProjectApplicationViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectApplicationSerializer + queryset = models.ProjectApplication.objects.all() + + +class ProjectApplicationLocationsProposalOfEoiSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectApplicationLocationsProposalOfEoi + exclude = () + + +class ProjectApplicationLocationsProposalOfEoiViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectApplicationLocationsProposalOfEoiSerializer + queryset = models.ProjectApplicationLocationsProposalOfEoi.objects.all() + + +class ProjectApplicationfeedbackSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectApplicationfeedback + exclude = () + + +class ProjectApplicationfeedbackViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectApplicationfeedbackSerializer + queryset = models.ProjectApplicationfeedback.objects.all() + + +class ProjectAssessmentSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectAssessment + exclude = () + + +class ProjectAssessmentViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectAssessmentSerializer + queryset = models.ProjectAssessment.objects.all() + + +class ProjectClarificationrequestanswerfileSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectClarificationrequestanswerfile + exclude = () + + +class ProjectClarificationrequestanswerfileViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectClarificationrequestanswerfileSerializer + queryset = models.ProjectClarificationrequestanswerfile.objects.all() + + +class ProjectClarificationrequestquestionSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectClarificationrequestquestion + exclude = () + + +class ProjectClarificationrequestquestionViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectClarificationrequestquestionSerializer + queryset = models.ProjectClarificationrequestquestion.objects.all() + + +class ProjectEoiSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectEoi + exclude = () + + +class ProjectEoiViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectEoiSerializer + queryset = models.ProjectEoi.objects.all() + + +class ProjectEoiFocalPointsSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectEoiFocalPoints + exclude = () + + +class ProjectEoiFocalPointsViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectEoiFocalPointsSerializer + queryset = models.ProjectEoiFocalPoints.objects.all() + + +class ProjectEoiInvitedPartnersSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectEoiInvitedPartners + exclude = () + + +class ProjectEoiInvitedPartnersViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectEoiInvitedPartnersSerializer + queryset = models.ProjectEoiInvitedPartners.objects.all() + + +class ProjectEoiLocationsSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectEoiLocations + exclude = () + + +class ProjectEoiLocationsViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectEoiLocationsSerializer + queryset = models.ProjectEoiLocations.objects.all() + + +class ProjectEoiReviewersSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectEoiReviewers + exclude = () + + +class ProjectEoiReviewersViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectEoiReviewersSerializer + queryset = models.ProjectEoiReviewers.objects.all() + + +class ProjectEoiSpecializationsSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectEoiSpecializations + exclude = () + + +class ProjectEoiSpecializationsViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectEoiSpecializationsSerializer + queryset = models.ProjectEoiSpecializations.objects.all() + + +class ProjectEoiattachmentSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectEoiattachment + exclude = () + + +class ProjectEoiattachmentViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectEoiattachmentSerializer + queryset = models.ProjectEoiattachment.objects.all() + + +class ProjectPinSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProjectPin + exclude = () + + +class ProjectPinViewSet(URFReadOnlyModelViewSet): + serializer_class = ProjectPinSerializer + queryset = models.ProjectPin.objects.all() + + +class ReviewPartnerflagSerializer(serializers.ModelSerializer): + class Meta: + model = models.ReviewPartnerflag + exclude = () + + +class ReviewPartnerflagViewSet(URFReadOnlyModelViewSet): + serializer_class = ReviewPartnerflagSerializer + queryset = models.ReviewPartnerflag.objects.all() + + +class ReviewPartnerverificationSerializer(serializers.ModelSerializer): + class Meta: + model = models.ReviewPartnerverification + exclude = () + + +class ReviewPartnerverificationViewSet(URFReadOnlyModelViewSet): + serializer_class = ReviewPartnerverificationSerializer + queryset = models.ReviewPartnerverification.objects.all() + + +class SanctionslistSanctioneditemSerializer(serializers.ModelSerializer): + class Meta: + model = models.SanctionslistSanctioneditem + exclude = () + + +class SanctionslistSanctioneditemViewSet(URFReadOnlyModelViewSet): + serializer_class = SanctionslistSanctioneditemSerializer + queryset = models.SanctionslistSanctioneditem.objects.all() + + +class SanctionslistSanctionednameSerializer(serializers.ModelSerializer): + class Meta: + model = models.SanctionslistSanctionedname + exclude = () + + +class SanctionslistSanctionednameViewSet(URFReadOnlyModelViewSet): + serializer_class = SanctionslistSanctionednameSerializer + queryset = models.SanctionslistSanctionedname.objects.all() + + +class SanctionslistSanctionednamematchSerializer(serializers.ModelSerializer): + class Meta: + model = models.SanctionslistSanctionednamematch + exclude = () + + +class SanctionslistSanctionednamematchViewSet(URFReadOnlyModelViewSet): + serializer_class = SanctionslistSanctionednamematchSerializer + queryset = models.SanctionslistSanctionednamematch.objects.all() + + +class SequencesSequenceSerializer(serializers.ModelSerializer): + class Meta: + model = models.SequencesSequence + exclude = () + + +class SequencesSequenceViewSet(URFReadOnlyModelViewSet): + serializer_class = SequencesSequenceSerializer + queryset = models.SequencesSequence.objects.all() diff --git a/src/etools_datamart/apps/sources/unpp/api_urls.py b/src/etools_datamart/apps/sources/unpp/api_urls.py new file mode 100644 index 000000000..53b4a64b0 --- /dev/null +++ b/src/etools_datamart/apps/sources/unpp/api_urls.py @@ -0,0 +1,62 @@ +from etools_datamart.api.urls import router + +from . import api + +router.register(r'unpp/agencyagency', api.AgencyAgencyViewSet) +router.register(r'unpp/agencyagencymember', api.AgencyAgencymemberViewSet) +router.register(r'unpp/agencyagencyoffice', api.AgencyAgencyofficeViewSet) +router.register(r'unpp/agencyagencyprofile', api.AgencyAgencyprofileViewSet) +router.register(r'unpp/agencyotheragency', api.AgencyOtheragencyViewSet) +router.register(r'unpp/commonadminlevel1', api.CommonAdminlevel1ViewSet) +router.register(r'unpp/commoncommonfile', api.CommonCommonfileViewSet) +router.register(r'unpp/commonpoint', api.CommonPointViewSet) +router.register(r'unpp/commonsector', api.CommonSectorViewSet) +router.register(r'unpp/commonspecialization', api.CommonSpecializationViewSet) +router.register(r'unpp/externalspartnervendornumber', api.ExternalsPartnervendornumberViewSet) +router.register(r'unpp/externalsunicefvendordata', api.ExternalsUnicefvendordataViewSet) +router.register(r'unpp/notificationnotification', api.NotificationNotificationViewSet) +router.register(r'unpp/notificationnotifieduser', api.NotificationNotifieduserViewSet) +router.register(r'unpp/partnerpartner', api.PartnerPartnerViewSet) +router.register(r'unpp/partnerpartnerlocationfieldoffices', api.PartnerPartnerLocationFieldOfficesViewSet) +router.register(r'unpp/partnerpartnerauditassessment', api.PartnerPartnerauditassessmentViewSet) +router.register(r'unpp/partnerpartnerauditreport', api.PartnerPartnerauditreportViewSet) +router.register(r'unpp/partnerpartnerauthorisedofficer', api.PartnerPartnerauthorisedofficerViewSet) +router.register(r'unpp/partnerpartnerbudget', api.PartnerPartnerbudgetViewSet) +router.register(r'unpp/partnerpartnercapacityassessment', api.PartnerPartnercapacityassessmentViewSet) +router.register(r'unpp/partnerpartnercollaborationevidence', api.PartnerPartnercollaborationevidenceViewSet) +router.register(r'unpp/partnerpartnercollaborationpartnership', api.PartnerPartnercollaborationpartnershipViewSet) +router.register(r'unpp/partnerpartnerdirector', api.PartnerPartnerdirectorViewSet) +router.register(r'unpp/partnerpartnerexperience', api.PartnerPartnerexperienceViewSet) +router.register(r'unpp/partnerpartnerfunding', api.PartnerPartnerfundingViewSet) +router.register(r'unpp/partnerpartnergoverningdocument', api.PartnerPartnergoverningdocumentViewSet) +router.register(r'unpp/partnerpartnerheadorganization', api.PartnerPartnerheadorganizationViewSet) +router.register(r'unpp/partnerpartnerinternalcontrol', api.PartnerPartnerinternalcontrolViewSet) +router.register(r'unpp/partnerpartnermailingaddress', api.PartnerPartnermailingaddressViewSet) +router.register(r'unpp/partnerpartnermandatemission', api.PartnerPartnermandatemissionViewSet) +router.register(r'unpp/partnerpartnermember', api.PartnerPartnermemberViewSet) +router.register(r'unpp/partnerpartnerotherinfo', api.PartnerPartnerotherinfoViewSet) +router.register(r'unpp/partnerpartnerpolicyarea', api.PartnerPartnerpolicyareaViewSet) +router.register(r'unpp/partnerpartnerprofile', api.PartnerPartnerprofileViewSet) +router.register(r'unpp/partnerpartnerregistrationdocument', api.PartnerPartnerregistrationdocumentViewSet) +router.register(r'unpp/partnerpartnerreporting', api.PartnerPartnerreportingViewSet) +router.register(r'unpp/partnerpartnerreview', api.PartnerPartnerreviewViewSet) +router.register(r'unpp/projectapplication', api.ProjectApplicationViewSet) +router.register(r'unpp/projectapplicationlocationsproposalofeoi', api.ProjectApplicationLocationsProposalOfEoiViewSet) +router.register(r'unpp/projectapplicationfeedback', api.ProjectApplicationfeedbackViewSet) +router.register(r'unpp/projectassessment', api.ProjectAssessmentViewSet) +router.register(r'unpp/projectclarificationrequestanswerfile', api.ProjectClarificationrequestanswerfileViewSet) +router.register(r'unpp/projectclarificationrequestquestion', api.ProjectClarificationrequestquestionViewSet) +router.register(r'unpp/projecteoi', api.ProjectEoiViewSet) +router.register(r'unpp/projecteoifocalpoints', api.ProjectEoiFocalPointsViewSet) +router.register(r'unpp/projecteoiinvitedpartners', api.ProjectEoiInvitedPartnersViewSet) +router.register(r'unpp/projecteoilocations', api.ProjectEoiLocationsViewSet) +router.register(r'unpp/projecteoireviewers', api.ProjectEoiReviewersViewSet) +router.register(r'unpp/projecteoispecializations', api.ProjectEoiSpecializationsViewSet) +router.register(r'unpp/projecteoiattachment', api.ProjectEoiattachmentViewSet) +router.register(r'unpp/projectpin', api.ProjectPinViewSet) +router.register(r'unpp/reviewpartnerflag', api.ReviewPartnerflagViewSet) +router.register(r'unpp/reviewpartnerverification', api.ReviewPartnerverificationViewSet) +router.register(r'unpp/sanctionslistsanctioneditem', api.SanctionslistSanctioneditemViewSet) +router.register(r'unpp/sanctionslistsanctionedname', api.SanctionslistSanctionednameViewSet) +router.register(r'unpp/sanctionslistsanctionednamematch', api.SanctionslistSanctionednamematchViewSet) +router.register(r'unpp/sequencessequence', api.SequencesSequenceViewSet) diff --git a/src/etools_datamart/apps/sources/unpp/apps.py b/src/etools_datamart/apps/sources/unpp/apps.py new file mode 100644 index 000000000..ae9a0e722 --- /dev/null +++ b/src/etools_datamart/apps/sources/unpp/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + + +class Config(AppConfig): + name = 'etools_datamart.apps.sources.unpp' + verbose_name = "UNPP" + label = 'unpp' + + def ready(self): + pass + # from .enrichment import patch + # patch() diff --git a/src/etools_datamart/apps/sources/unpp/management/__init_.py b/src/etools_datamart/apps/sources/unpp/management/__init_.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/sources/unpp/management/commands/__init_.py b/src/etools_datamart/apps/sources/unpp/management/commands/__init_.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/sources/unpp/management/commands/inspectunpp.py b/src/etools_datamart/apps/sources/unpp/management/commands/inspectunpp.py new file mode 100644 index 000000000..a2c2470ec --- /dev/null +++ b/src/etools_datamart/apps/sources/unpp/management/commands/inspectunpp.py @@ -0,0 +1,308 @@ +import logging +import re +from collections import OrderedDict +from datetime import datetime +from pathlib import Path + +from django.core.management import call_command, CommandError +from django.core.management.commands.inspectdb import Command as BaseCommand +from django.db import connections + +from django_regex.utils import RegexList + +logger = logging.getLogger(__name__) + +INGNORED_TABLES = RegexList([ + 'account_user_user_permissions', + 'auth_permission', + 'auth_group_permissions', + 'authtoken_token', + 'background_task', + 'django_admin_log', + 'django_migrations', + 'django_session', + 'social_auth_.*' +]) +NO_API = RegexList([ + 'AccountUserGroups', + 'AccountUser', + 'AccountUserUserPermissions', + 'AccountUserprofile', + 'Auth.*', + 'Django.*' +]) + +REVERSE_RELATION_NAMES = { + # '{model_name}.{att_name}': 'related_name' +} + +APPS = [ + 'Account', + 'Agency', + 'Common', + 'Django', + 'Externals', + 'Notification', + 'Partner', + 'Project', + 'Review', + 'Sanctionslist', +] + + +class Command(BaseCommand): + args = '' + help = "Introspects UNPP database outputs a etools_datamart.apps.unpp.models module." + db_module = 'django.contrib.gis.db' + + def add_arguments(self, parser): + parser.add_argument( + '--app_label', + default='unpp', + type=str, + help='application name', + ) + + def handle(self, *args, **options): + options['database'] = 'unpp' + options['include_partitions'] = False + options['include_views'] = False + options['table'] = None + options['table_name_filter'] = lambda table_name: table_name not in INGNORED_TABLES + self.unpp_models = [] + + try: + output_file = Path(__file__).parent.parent.parent / 'models.py' + if output_file.exists(): + output_file.rename(output_file.with_suffix('.bak')) + with output_file.open('w') as output: + for line in self.handle_inspection(options): + output.write("%s\n" % line) + self.handle_admin(options) + self.handle_api(options) + call_command('check') + except NotImplementedError: + raise CommandError( + "Database inspection isn't supported for the currently selected database backend.", + ) + + def handle_model_admin(self, model_name): + yield "" + yield "@register(models.%s)" % model_name + yield "class %sAdmin(DatamartSourceModelAdmin, ModelAdmin):" % model_name + yield " list_filter = []" + yield "" + + def handle_admin(self, options): + output_file = Path(__file__).parent.parent.parent / 'admin.py' + if output_file.exists(): + output_file.rename(output_file.with_suffix('.bak')) + with output_file.open('w') as output: + output.write("from django.contrib.admin import ModelAdmin, register\n\n") + output.write("from etools_datamart.apps.core.admin_mixins " + "import DatamartSourceModelAdmin\n\n") + output.write("from . import models\n\n") + + for model_name in self.unpp_models: + for line in self.handle_model_admin(model_name): + output.write("%s\n" % line) + + def handle_api(self, options): + verbosity = options['verbosity'] + + api_file = Path(__file__).parent.parent.parent / 'api.py' + urls_file = Path(__file__).parent.parent.parent / 'api_urls.py' + if urls_file.exists(): + urls_file.rename(urls_file.with_suffix('.bak')) + if api_file.exists(): + api_file.rename(api_file.with_suffix('.bak')) + with urls_file.open('w') as urls: + urls.write("from etools_datamart.api.urls import router\n\n") + urls.write("from . import api\n\n") + with api_file.open('w') as api: + api.write("from unicef_rest_framework.views import URFReadOnlyModelViewSet\n\n") + api.write("from etools_datamart.api.endpoints.etools import serializers\n") + api.write("from etools_datamart.apps.sources.unpp import models\n") + + for model_name in self.unpp_models: + if model_name not in NO_API: + if verbosity >= 2: + self.stdout.write("Generating endpoint for %s" % model_name) + for line in self.handle_viewset(model_name): + api.write("%s\n" % line) + line = "router.register(r'unpp/%s', api.%sViewSet)" % (model_name.lower(), model_name) + urls.write("%s\n" % line) + else: + if verbosity >= 1: + self.stdout.write("Ignoring endpoint for %s" % model_name) + + def handle_viewset(self, model_name): + yield "" + yield "" + yield "class %sSerializer(serializers.ModelSerializer):" % model_name + yield " class Meta:" + yield " model = models.%s" % model_name + yield " exclude = ()" + yield "" + yield "" + yield "class %sViewSet(URFReadOnlyModelViewSet):" % model_name + yield " serializer_class = %sSerializer" % model_name + yield " queryset = models.%s.objects.all()" % model_name + + def handle_inspection(self, options): + connection = connections[options['database']] + # 'table_name_filter' is a stealth option + table_name_filter = options.get('table_name_filter') + app_label = options.get('app_label') + + def table2model(table_name): + return re.sub(r'[^a-zA-Z0-9]', '', table_name.title()) + + with connection.cursor() as cursor: + yield "# flake8: noqa F405." + yield "# This is an auto-generated UNPP model module." + yield "# Generated on %s" % datetime.now() + yield 'from %s import models\n' % self.db_module + yield 'from etools_datamart.apps.core.readonly import ReadOnlyModel' + known_models = [] + table_info = connection.introspection.get_table_list(cursor) + + # Determine types of tables and/or views to be introspected. + types = {'t'} + if options['include_partitions']: + types.add('p') + if options['include_views']: + types.add('v') + + for table_name in (options['table'] or sorted(info.name for info in table_info if info.type in types)): + if table_name_filter is not None and callable(table_name_filter): + if not table_name_filter(table_name): + continue + model_name = table2model(table_name) + self.unpp_models.append(model_name) + try: + try: + relations = connection.introspection.get_relations(cursor, table_name) + except NotImplementedError: + relations = {} + try: + constraints = connection.introspection.get_constraints(cursor, table_name) + except NotImplementedError: + constraints = {} + primary_key_column = connection.introspection.get_primary_key_column(cursor, table_name) + unique_columns = [ + c['columns'][0] for c in constraints.values() + if c['unique'] and len(c['columns']) == 1 + ] + table_description = connection.introspection.get_table_description(cursor, table_name) + except Exception as e: + yield "# Unable to inspect table '%s'" % table_name + yield "# The error was: %s" % e + continue + + yield '' + yield '' + yield 'class %s(ReadOnlyModel):' % table2model(table_name) + known_models.append(table2model(table_name)) + used_column_names = [] # Holds column names used in the table so far + column_to_field_name = {} # Maps column names to names of model fields + for row in table_description: + comment_notes = [] # Holds Field notes, to be displayed in a Python comment. + extra_params = OrderedDict() # Holds Field parameters such as 'db_column'. + column_name = row.name + is_relation = column_name in relations + + att_name, params, notes = self.normalize_col_name( + column_name, used_column_names, is_relation) + extra_params.update(params) + comment_notes.extend(notes) + + used_column_names.append(att_name) + column_to_field_name[column_name] = att_name + if column_name == 'id': + continue + # Add primary_key and unique, if necessary. + if column_name == primary_key_column: + extra_params['primary_key'] = True + elif column_name in unique_columns: + extra_params['unique'] = True + + if is_relation: + rel_to = ( + "self" if relations[column_name][1] == table_name + else table2model(relations[column_name][1]) + ) + # if rel_to in known_models: + # field_type = 'ForeignKey(%s' % rel_to + # else: + # field_type = "ForeignKey('%s'" % rel_to + if 'unique' in extra_params: + extra_params.pop('unique') + ftype = 'OneToOneField' + elif 'primary_key' in extra_params: + extra_params.pop('primary_key') + ftype = 'OneToOneField' + else: + ftype = 'ForeignKey' + + if rel_to in known_models: + field_type = f"{ftype}({rel_to}" + else: + if rel_to == 'self': + field_type = f"{ftype}('self'" + else: + field_type = f"{ftype}('{app_label}.{rel_to}'" + + else: + # Calling `get_field_type` to get the field type string and any + # additional parameters and notes. + field_type, field_params, field_notes = self.get_field_type(connection, table_name, row) + extra_params.update(field_params) + comment_notes.extend(field_notes) + + field_type += '(' + + # Don't output 'id = meta.AutoField(primary_key=True)', because + # that's assumed if it doesn't exist. + if att_name == 'id' and extra_params == {'primary_key': True}: + if field_type == 'AutoField(': + continue + elif field_type == 'IntegerField(' and not connection.features.can_introspect_autofield: + comment_notes.append('AutoField?') + + # Add 'null' and 'blank', if the 'null_ok' flag was present + # in the table description. + if row.null_ok: # If it's NULL... + extra_params['blank'] = True + extra_params['null'] = True + + field_desc = '%s = %s%s' % ( + att_name, + # Custom fields will have a dotted path + # '' if '.' in field_type else 'models.', + 'models.', + field_type, + ) + if field_type.startswith('ForeignKey(') or field_type.startswith('OneToOneField('): + _related_name = f'{model_name}.{att_name}' + + if _related_name in REVERSE_RELATION_NAMES: + _related_name = REVERSE_RELATION_NAMES[_related_name] + else: + _related_name = _related_name.replace('.', '_') + field_desc += ', models.PROTECT' + field_desc += f", related_name='{_related_name}'" + + if extra_params: + if not field_desc.endswith('('): + field_desc += ', ' + field_desc += ', '.join('%s=%r' % (k, v) for k, v in extra_params.items()) + field_desc += ')' + if comment_notes: + field_desc += ' # ' + ' '.join(comment_notes) + yield ' %s' % field_desc + is_view = any(info.name == table_name and info.type == 'v' for info in table_info) + is_partition = any(info.name == table_name and info.type == 'p' for info in table_info) + for meta_line in self.get_meta(table_name, constraints, column_to_field_name, is_view, is_partition): + yield meta_line + yield ' app_label = %r' % app_label diff --git a/src/etools_datamart/apps/sources/unpp/models.py b/src/etools_datamart/apps/sources/unpp/models.py new file mode 100644 index 000000000..919f4fef2 --- /dev/null +++ b/src/etools_datamart/apps/sources/unpp/models.py @@ -0,0 +1,1032 @@ +# flake8: noqa F405. +# This is an auto-generated UNPP model module. +# Generated on 2020-06-01 16:55:50.239245 +from django.contrib.gis.db import models + +from etools_datamart.apps.core.readonly import ReadOnlyModel + + +class AccountUser(ReadOnlyModel): + password = models.CharField(max_length=128) + last_login = models.DateTimeField(blank=True, null=True) + is_superuser = models.BooleanField() + email = models.CharField(unique=True, max_length=254) + is_staff = models.BooleanField() + is_active = models.BooleanField() + date_joined = models.DateTimeField() + fullname = models.CharField(max_length=512, blank=True, null=True) + + class Meta: + managed = False + db_table = 'account_user' + app_label = 'unpp' + + +class AccountUserGroups(ReadOnlyModel): + user = models.ForeignKey(AccountUser, models.PROTECT, related_name='AccountUserGroups_user') + group = models.ForeignKey('unpp.AuthGroup', models.PROTECT, related_name='AccountUserGroups_group') + + class Meta: + managed = False + db_table = 'account_user_groups' + unique_together = (('user', 'group'),) + app_label = 'unpp' + + +class AccountUserprofile(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + user = models.OneToOneField(AccountUser, models.PROTECT, related_name='AccountUserprofile_user') + notification_frequency = models.TextField(blank=True, null=True) + accepted_tos = models.BooleanField() + + class Meta: + managed = False + db_table = 'account_userprofile' + app_label = 'unpp' + + +class AgencyAgency(ReadOnlyModel): + name = models.CharField(unique=True, max_length=255) + + class Meta: + managed = False + db_table = 'agency_agency' + app_label = 'unpp' + + +class AgencyAgencymember(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + office = models.ForeignKey('unpp.AgencyAgencyoffice', models.PROTECT, related_name='AgencyAgencymember_office') + user = models.ForeignKey(AccountUser, models.PROTECT, related_name='AgencyAgencymember_user') + telephone = models.CharField(max_length=255, blank=True, null=True) + role = models.TextField() + + class Meta: + managed = False + db_table = 'agency_agencymember' + unique_together = (('user', 'office'),) + app_label = 'unpp' + + +class AgencyAgencyoffice(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='AgencyAgencyoffice_agency') + country = models.CharField(max_length=2) + + class Meta: + managed = False + db_table = 'agency_agencyoffice' + unique_together = (('agency', 'country'),) + app_label = 'unpp' + + +class AgencyAgencyprofile(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + eoi_template = models.CharField(max_length=100) + agency = models.OneToOneField(AgencyAgency, models.PROTECT, related_name='AgencyAgencyprofile_agency') + + class Meta: + managed = False + db_table = 'agency_agencyprofile' + app_label = 'unpp' + + +class AgencyOtheragency(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + name = models.CharField(max_length=255) + + class Meta: + managed = False + db_table = 'agency_otheragency' + app_label = 'unpp' + + +class AuthGroup(ReadOnlyModel): + name = models.CharField(unique=True, max_length=80) + + class Meta: + managed = False + db_table = 'auth_group' + app_label = 'unpp' + + +class CommonAdminlevel1(ReadOnlyModel): + name = models.CharField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=3) + + class Meta: + managed = False + db_table = 'common_adminlevel1' + unique_together = (('name', 'country_code'),) + app_label = 'unpp' + + +class CommonCommonfile(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + file_field = models.CharField(max_length=100) + + class Meta: + managed = False + db_table = 'common_commonfile' + app_label = 'unpp' + + +class CommonPoint(ReadOnlyModel): + lat = models.DecimalField(max_digits=8, decimal_places=5, blank=True, null=True) + lon = models.DecimalField(max_digits=8, decimal_places=5, blank=True, null=True) + admin_level_1 = models.ForeignKey(CommonAdminlevel1, models.PROTECT, related_name='CommonPoint_admin_level_1') + + class Meta: + managed = False + db_table = 'common_point' + app_label = 'unpp' + + +class CommonSector(ReadOnlyModel): + name = models.CharField(max_length=255) + + class Meta: + managed = False + db_table = 'common_sector' + app_label = 'unpp' + + +class CommonSpecialization(ReadOnlyModel): + name = models.CharField(max_length=255) + category = models.ForeignKey(CommonSector, models.PROTECT, related_name='CommonSpecialization_category') + + class Meta: + managed = False + db_table = 'common_specialization' + app_label = 'unpp' + + +class DjangoContentType(ReadOnlyModel): + app_label = models.CharField(max_length=100) + model = models.CharField(max_length=100) + + class Meta: + managed = False + db_table = 'django_content_type' + unique_together = (('app_label', 'model'),) + app_label = 'unpp' + + +class DjangoSite(ReadOnlyModel): + domain = models.CharField(unique=True, max_length=100) + name = models.CharField(max_length=50) + + class Meta: + managed = False + db_table = 'django_site' + app_label = 'unpp' + + +class ExternalsPartnervendornumber(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + number = models.TextField() + agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='ExternalsPartnervendornumber_agency') + partner = models.ForeignKey('unpp.PartnerPartner', models.PROTECT, related_name='ExternalsPartnervendornumber_partner') + business_area = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'externals_partnervendornumber' + unique_together = (('agency', 'partner', 'business_area', 'number'),) + app_label = 'unpp' + + +class ExternalsUnicefvendordata(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + business_area = models.TextField() + vendor_number = models.TextField() + vendor_name = models.TextField() + total_cash_transfers = models.FloatField() + cash_transfers_this_year = models.FloatField() + year = models.IntegerField() + + class Meta: + managed = False + db_table = 'externals_unicefvendordata' + unique_together = (('business_area', 'vendor_number', 'year'),) + app_label = 'unpp' + + +class NotificationNotification(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + name = models.CharField(max_length=255) + description = models.TextField() + source = models.TextField() + content_type = models.ForeignKey(DjangoContentType, models.PROTECT, related_name='NotificationNotification_content_type', blank=True, null=True) + object_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'notification_notification' + app_label = 'unpp' + + +class NotificationNotifieduser(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + did_read = models.BooleanField() + notification = models.ForeignKey(NotificationNotification, models.PROTECT, related_name='NotificationNotifieduser_notification') + recipient = models.ForeignKey(AccountUser, models.PROTECT, related_name='NotificationNotifieduser_recipient') + sent_as_email = models.BooleanField() + + class Meta: + managed = False + db_table = 'notification_notifieduser' + unique_together = (('notification', 'recipient'),) + app_label = 'unpp' + + +class PartnerPartner(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + legal_name = models.TextField() + display_type = models.CharField(max_length=3) + country_code = models.CharField(max_length=2) + is_active = models.BooleanField() + hq = models.ForeignKey('self', models.PROTECT, related_name='PartnerPartner_hq', blank=True, null=True) + country_presence = models.TextField(blank=True, null=True) # This field type is a guess. + staff_globally = models.CharField(max_length=3, blank=True, null=True) + engagement_operate_desc = models.TextField(blank=True, null=True) + staff_in_country = models.CharField(max_length=3, blank=True, null=True) + location_of_office = models.ForeignKey(CommonPoint, models.PROTECT, related_name='PartnerPartner_location_of_office', blank=True, null=True) + more_office_in_country = models.BooleanField(blank=True, null=True) + is_locked = models.BooleanField() + declaration = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartner_declaration', blank=True, null=True) + migrated_from = models.TextField(blank=True, null=True) + migrated_original_id = models.IntegerField(blank=True, null=True) + migrated_timestamp = models.DateTimeField(blank=True, null=True) + legal_name_length = models.IntegerField() + + class Meta: + managed = False + db_table = 'partner_partner' + unique_together = (('legal_name', 'country_code', 'hq'),) + app_label = 'unpp' + + +class PartnerPartnerLocationFieldOffices(ReadOnlyModel): + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerLocationFieldOffices_partner') + point = models.ForeignKey(CommonPoint, models.PROTECT, related_name='PartnerPartnerLocationFieldOffices_point') + + class Meta: + managed = False + db_table = 'partner_partner_location_field_offices' + unique_together = (('partner', 'point'),) + app_label = 'unpp' + + +class PartnerPartnerauditassessment(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + regular_audited = models.BooleanField(blank=True, null=True) + major_accountability_issues_highlighted = models.BooleanField(blank=True, null=True) + comment = models.TextField(blank=True, null=True) + partner = models.OneToOneField(PartnerPartner, models.PROTECT, related_name='PartnerPartnerauditassessment_partner') + regular_audited_comment = models.TextField(blank=True, null=True) + regular_capacity_assessments = models.BooleanField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerauditassessment' + app_label = 'unpp' + + +class PartnerPartnerauditreport(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + org_audit = models.CharField(max_length=3, blank=True, null=True) + link_report = models.CharField(max_length=200, blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerauditreport_partner') + most_recent_audit_report = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnerauditreport_most_recent_audit_report', blank=True, null=True) + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerauditreport_created_by', blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerauditreport' + app_label = 'unpp' + + +class PartnerPartnerauthorisedofficer(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + job_title = models.CharField(max_length=255, blank=True, null=True) + telephone = models.CharField(max_length=255, blank=True, null=True) + fax = models.CharField(max_length=255, blank=True, null=True) + email = models.CharField(max_length=255, blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerauthorisedofficer_partner') + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerauthorisedofficer_created_by', blank=True, null=True) + fullname = models.CharField(max_length=512, blank=True, null=True) + mobile = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerauthorisedofficer' + app_label = 'unpp' + + +class PartnerPartnerbudget(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + year = models.SmallIntegerField() + budget = models.CharField(max_length=3, blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerbudget_partner') + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerbudget_created_by', blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerbudget' + unique_together = (('partner', 'year'),) + app_label = 'unpp' + + +class PartnerPartnercapacityassessment(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + assessment_type = models.TextField(blank=True, null=True) + report_url = models.CharField(max_length=200, blank=True, null=True) + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnercapacityassessment_created_by', blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnercapacityassessment_partner') + report_file = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnercapacityassessment_report_file', blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnercapacityassessment' + app_label = 'unpp' + + +class PartnerPartnercollaborationevidence(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + mode = models.CharField(max_length=3, blank=True, null=True) + organization_name = models.CharField(max_length=1000, blank=True, null=True) + date_received = models.DateField(blank=True, null=True) + evidence_file = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnercollaborationevidence_evidence_file', blank=True, null=True) + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnercollaborationevidence_created_by') + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnercollaborationevidence_partner') + editable = models.BooleanField() + + class Meta: + managed = False + db_table = 'partner_partnercollaborationevidence' + app_label = 'unpp' + + +class PartnerPartnercollaborationpartnership(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + description = models.CharField(max_length=10000, blank=True, null=True) + partner_number = models.CharField(max_length=200, blank=True, null=True) + agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='PartnerPartnercollaborationpartnership_agency', blank=True, null=True) + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnercollaborationpartnership_created_by') + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnercollaborationpartnership_partner') + + class Meta: + managed = False + db_table = 'partner_partnercollaborationpartnership' + unique_together = (('partner', 'agency'),) + app_label = 'unpp' + + +class PartnerPartnerdirector(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + job_title = models.CharField(max_length=255, blank=True, null=True) + authorized = models.BooleanField(blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerdirector_partner') + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerdirector_created_by', blank=True, null=True) + fullname = models.CharField(max_length=512, blank=True, null=True) + email = models.CharField(max_length=255, blank=True, null=True) + fax = models.CharField(max_length=255, blank=True, null=True) + telephone = models.CharField(max_length=255, blank=True, null=True) + mobile = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerdirector' + app_label = 'unpp' + + +class PartnerPartnerexperience(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + years = models.CharField(max_length=3, blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerexperience_partner') + specialization = models.ForeignKey(CommonSpecialization, models.PROTECT, related_name='PartnerPartnerexperience_specialization', blank=True, null=True) + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerexperience_created_by', blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerexperience' + app_label = 'unpp' + + +class PartnerPartnerfunding(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + source_core_funding = models.CharField(max_length=5000) + major_donors = models.TextField(blank=True, null=True) # This field type is a guess. + main_donors_list = models.CharField(max_length=5000, blank=True, null=True) + partner = models.OneToOneField(PartnerPartner, models.PROTECT, related_name='PartnerPartnerfunding_partner') + + class Meta: + managed = False + db_table = 'partner_partnerfunding' + app_label = 'unpp' + + +class PartnerPartnergoverningdocument(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + editable = models.BooleanField() + document = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnergoverningdocument_document') + profile = models.ForeignKey('unpp.PartnerPartnerprofile', models.PROTECT, related_name='PartnerPartnergoverningdocument_profile') + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnergoverningdocument_created_by') + + class Meta: + managed = False + db_table = 'partner_partnergoverningdocument' + app_label = 'unpp' + + +class PartnerPartnerheadorganization(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + email = models.CharField(max_length=255, blank=True, null=True) + job_title = models.CharField(max_length=255, blank=True, null=True) + telephone = models.CharField(max_length=255, blank=True, null=True) + fax = models.CharField(max_length=255, blank=True, null=True) + mobile = models.CharField(max_length=255, blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerheadorganization_partner', blank=True, null=True) + fullname = models.CharField(max_length=512, blank=True, null=True) + authorized = models.BooleanField(blank=True, null=True) + board_member = models.BooleanField(blank=True, null=True) + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerheadorganization_created_by', blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerheadorganization' + app_label = 'unpp' + + +class PartnerPartnerinternalcontrol(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + functional_responsibility = models.CharField(max_length=3) + segregation_duties = models.BooleanField(blank=True, null=True) + comment = models.TextField(blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerinternalcontrol_partner') + + class Meta: + managed = False + db_table = 'partner_partnerinternalcontrol' + unique_together = (('partner', 'functional_responsibility'),) + app_label = 'unpp' + + +class PartnerPartnermailingaddress(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + mailing_type = models.CharField(max_length=3) + street = models.CharField(max_length=1000, blank=True, null=True) + po_box = models.CharField(max_length=1000, blank=True, null=True) + city = models.CharField(max_length=1000, blank=True, null=True) + country = models.CharField(max_length=2, blank=True, null=True) + zip_code = models.CharField(max_length=1000, blank=True, null=True) + telephone = models.CharField(max_length=1000, blank=True, null=True) + fax = models.CharField(max_length=1000, blank=True, null=True) + website = models.CharField(max_length=1000, blank=True, null=True) + org_email = models.CharField(max_length=1000, blank=True, null=True) + partner = models.OneToOneField(PartnerPartner, models.PROTECT, related_name='PartnerPartnermailingaddress_partner') + + class Meta: + managed = False + db_table = 'partner_partnermailingaddress' + app_label = 'unpp' + + +class PartnerPartnermandatemission(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + background_and_rationale = models.TextField(blank=True, null=True) + mandate_and_mission = models.TextField(blank=True, null=True) + governance_structure = models.TextField(blank=True, null=True) + governance_hq = models.TextField(blank=True, null=True) + governance_organigram = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnermandatemission_governance_organigram', blank=True, null=True) + ethic_safeguard = models.BooleanField(blank=True, null=True) + ethic_safeguard_policy = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnermandatemission_ethic_safeguard_policy', blank=True, null=True) + ethic_fraud = models.BooleanField(blank=True, null=True) + ethic_fraud_policy = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnermandatemission_ethic_fraud_policy', blank=True, null=True) + population_of_concern = models.BooleanField(blank=True, null=True) + concern_groups = models.TextField(blank=True, null=True) # This field type is a guess. + security_high_risk_locations = models.BooleanField(blank=True, null=True) + security_high_risk_policy = models.BooleanField(blank=True, null=True) + security_desc = models.TextField(blank=True, null=True) + partner = models.OneToOneField(PartnerPartner, models.PROTECT, related_name='PartnerPartnermandatemission_partner') + ethic_fraud_comment = models.TextField(blank=True, null=True) + ethic_safeguard_comment = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnermandatemission' + app_label = 'unpp' + + +class PartnerPartnermember(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + title = models.CharField(max_length=255) + role = models.TextField() + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnermember_partner') + user = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnermember_user') + + class Meta: + managed = False + db_table = 'partner_partnermember' + unique_together = (('user', 'partner'),) + app_label = 'unpp' + + +class PartnerPartnerotherinfo(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + info_to_share = models.TextField(blank=True, null=True) + org_logo = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnerotherinfo_org_logo', blank=True, null=True) + confirm_data_updated = models.BooleanField(blank=True, null=True) + partner = models.OneToOneField(PartnerPartner, models.PROTECT, related_name='PartnerPartnerotherinfo_partner') + other_doc_1 = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnerotherinfo_other_doc_1', blank=True, null=True) + other_doc_2 = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnerotherinfo_other_doc_2', blank=True, null=True) + other_doc_3 = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnerotherinfo_other_doc_3', blank=True, null=True) + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerotherinfo_created_by', blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerotherinfo' + app_label = 'unpp' + + +class PartnerPartnerpolicyarea(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + area = models.CharField(max_length=3) + document_policies = models.BooleanField(blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerpolicyarea_partner') + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerpolicyarea_created_by', blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerpolicyarea' + unique_together = (('partner', 'area'),) + app_label = 'unpp' + + +class PartnerPartnerprofile(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + alias_name = models.CharField(max_length=255, blank=True, null=True) + legal_name_change = models.BooleanField(blank=True, null=True) + former_legal_name = models.CharField(max_length=255, blank=True, null=True) + working_languages = models.TextField(blank=True, null=True) # This field type is a guess. + working_languages_other = models.CharField(max_length=100, blank=True, null=True) + have_governing_document = models.BooleanField(blank=True, null=True) + have_management_approach = models.BooleanField(blank=True, null=True) + management_approach_desc = models.TextField(blank=True, null=True) + have_system_monitoring = models.BooleanField(blank=True, null=True) + system_monitoring_desc = models.TextField(blank=True, null=True) + have_feedback_mechanism = models.BooleanField(blank=True, null=True) + org_acc_system = models.CharField(max_length=3) + method_acc = models.CharField(max_length=3) + have_system_track = models.BooleanField(blank=True, null=True) + financial_control_system_desc = models.TextField(blank=True, null=True) + partner = models.OneToOneField(PartnerPartner, models.PROTECT, related_name='PartnerPartnerprofile_partner') + connectivity = models.BooleanField(blank=True, null=True) + connectivity_excuse = models.CharField(max_length=5000, blank=True, null=True) + experienced_staff = models.BooleanField(blank=True, null=True) + experienced_staff_desc = models.TextField(blank=True, null=True) + feedback_mechanism_desc = models.TextField(blank=True, null=True) + acronym = models.CharField(max_length=200, blank=True, null=True) + partnership_collaborate_institution = models.BooleanField(blank=True, null=True) + partnership_collaborate_institution_desc = models.CharField(max_length=5000, blank=True, null=True) + explain = models.TextField(blank=True, null=True) + have_bank_account = models.BooleanField(blank=True, null=True) + have_board_directors = models.BooleanField(blank=True, null=True) + have_separate_bank_account = models.BooleanField(blank=True, null=True) + missing_registration_document_comment = models.TextField(blank=True, null=True) + registered_to_operate_in_country = models.BooleanField(blank=True, null=True) + year_establishment = models.SmallIntegerField(blank=True, null=True) + have_authorised_officers = models.BooleanField(blank=True, null=True) + any_accreditation = models.BooleanField(blank=True, null=True) + any_partnered_with_un = models.BooleanField(blank=True, null=True) + any_reference = models.BooleanField(blank=True, null=True) + missing_governing_document_comment = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'partner_partnerprofile' + app_label = 'unpp' + + +class PartnerPartnerregistrationdocument(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + registration_number = models.TextField(blank=True, null=True) + editable = models.BooleanField() + issue_date = models.DateField() + expiry_date = models.DateField(blank=True, null=True) + document = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnerregistrationdocument_document') + profile = models.ForeignKey(PartnerPartnerprofile, models.PROTECT, related_name='PartnerPartnerregistrationdocument_profile') + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerregistrationdocument_created_by') + issuing_authority = models.TextField() + + class Meta: + managed = False + db_table = 'partner_partnerregistrationdocument' + app_label = 'unpp' + + +class PartnerPartnerreporting(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + key_result = models.TextField(blank=True, null=True) + publish_annual_reports = models.BooleanField(blank=True, null=True) + last_report = models.DateField(blank=True, null=True) + report = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnerreporting_report', blank=True, null=True) + link_report = models.CharField(max_length=1000, blank=True, null=True) + partner = models.OneToOneField(PartnerPartner, models.PROTECT, related_name='PartnerPartnerreporting_partner') + + class Meta: + managed = False + db_table = 'partner_partnerreporting' + app_label = 'unpp' + + +class PartnerPartnerreview(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + display_type = models.CharField(max_length=3) + performance_pm = models.CharField(max_length=3) + performance_financial = models.CharField(max_length=3) + performance_com_eng = models.CharField(max_length=3) + ethical_concerns = models.BooleanField(blank=True, null=True) + does_recommend = models.BooleanField(blank=True, null=True) + comment = models.TextField() + agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='PartnerPartnerreview_agency') + eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='PartnerPartnerreview_eoi') + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerreview_partner') + reviewer = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerreview_reviewer') + + class Meta: + managed = False + db_table = 'partner_partnerreview' + app_label = 'unpp' + + +class ProjectApplication(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + is_unsolicited = models.BooleanField() + status = models.CharField(max_length=3) + did_win = models.BooleanField() + did_accept = models.BooleanField() + ds_justification_select = models.TextField(blank=True, null=True) # This field type is a guess. + justification_reason = models.TextField(blank=True, null=True) + eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='ProjectApplication_eoi', blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ProjectApplication_partner') + submitter = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectApplication_submitter') + proposal_of_eoi_details = models.TextField() # This field type is a guess. + agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='ProjectApplication_agency') + did_withdraw = models.BooleanField() + withdraw_reason = models.TextField(blank=True, null=True) + did_decline = models.BooleanField() + eoi_converted = models.OneToOneField('unpp.ProjectEoi', models.PROTECT, related_name='ProjectApplication_eoi_converted', blank=True, null=True) + cn = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ProjectApplication_cn', blank=True, null=True) + accept_notification = models.OneToOneField(NotificationNotification, models.PROTECT, related_name='ProjectApplication_accept_notification', blank=True, null=True) + partner_decision_date = models.DateField(blank=True, null=True) + is_published = models.BooleanField(blank=True, null=True) + ds_attachment = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ProjectApplication_ds_attachment', blank=True, null=True) + published_timestamp = models.DateTimeField(blank=True, null=True) + partner_decision_maker = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectApplication_partner_decision_maker', blank=True, null=True) + agency_decision_date = models.DateField(blank=True, null=True) + agency_decision_maker = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectApplication_agency_decision_maker', blank=True, null=True) + + class Meta: + managed = False + db_table = 'project_application' + unique_together = (('eoi', 'partner'),) + app_label = 'unpp' + + +class ProjectApplicationLocationsProposalOfEoi(ReadOnlyModel): + application = models.ForeignKey(ProjectApplication, models.PROTECT, related_name='ProjectApplicationLocationsProposalOfEoi_application') + point = models.ForeignKey(CommonPoint, models.PROTECT, related_name='ProjectApplicationLocationsProposalOfEoi_point') + + class Meta: + managed = False + db_table = 'project_application_locations_proposal_of_eoi' + unique_together = (('application', 'point'),) + app_label = 'unpp' + + +class ProjectApplicationfeedback(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + feedback = models.TextField() + application = models.ForeignKey(ProjectApplication, models.PROTECT, related_name='ProjectApplicationfeedback_application') + provider = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectApplicationfeedback_provider') + + class Meta: + managed = False + db_table = 'project_applicationfeedback' + app_label = 'unpp' + + +class ProjectAssessment(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + date_reviewed = models.DateField() + application = models.ForeignKey(ProjectApplication, models.PROTECT, related_name='ProjectAssessment_application') + reviewer = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectAssessment_reviewer') + note = models.TextField(blank=True, null=True) + scores = models.TextField() # This field type is a guess. + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectAssessment_created_by') + modified_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectAssessment_modified_by', blank=True, null=True) + archived = models.BooleanField() + is_a_committee_score = models.BooleanField() + completed = models.BooleanField() + completed_date = models.DateField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'project_assessment' + unique_together = (('reviewer', 'application'),) + app_label = 'unpp' + + +class ProjectClarificationrequestanswerfile(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + title = models.TextField() + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectClarificationrequestanswerfile_created_by') + eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='ProjectClarificationrequestanswerfile_eoi') + file = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ProjectClarificationrequestanswerfile_file') + + class Meta: + managed = False + db_table = 'project_clarificationrequestanswerfile' + app_label = 'unpp' + + +class ProjectClarificationrequestquestion(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + question = models.TextField() + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectClarificationrequestquestion_created_by') + eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='ProjectClarificationrequestquestion_eoi') + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ProjectClarificationrequestquestion_partner') + + class Meta: + managed = False + db_table = 'project_clarificationrequestquestion' + app_label = 'unpp' + + +class ProjectEoi(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + display_type = models.CharField(max_length=3) + title = models.CharField(max_length=255) + cn_template = models.CharField(max_length=100, blank=True, null=True) + description = models.CharField(max_length=5000) + other_information = models.CharField(max_length=5000, blank=True, null=True) + start_date = models.DateField() + end_date = models.DateField() + deadline_date = models.DateField(blank=True, null=True) + notif_results_date = models.DateField(blank=True, null=True) + has_weighting = models.BooleanField() + justification = models.TextField(blank=True, null=True) + agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='ProjectEoi_agency') + agency_office = models.ForeignKey(AgencyAgencyoffice, models.PROTECT, related_name='ProjectEoi_agency_office') + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectEoi_created_by') + selected_source = models.CharField(max_length=3, blank=True, null=True) + completed_reason = models.TextField(blank=True, null=True) + goal = models.CharField(max_length=5000, blank=True, null=True) + assessments_criteria = models.TextField() # This field type is a guess. + completed_date = models.DateTimeField(blank=True, null=True) + review_summary_attachment = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ProjectEoi_review_summary_attachment', blank=True, null=True) + review_summary_comment = models.TextField(blank=True, null=True) + is_completed = models.BooleanField() + is_published = models.BooleanField() + completed_comment = models.TextField(blank=True, null=True) + completed_retention = models.TextField(blank=True, null=True) + sent_for_publishing = models.BooleanField() + published_timestamp = models.DateTimeField() + displayid = models.TextField(db_column='displayID', unique=True) # Field name made lowercase. + sent_for_decision = models.BooleanField() + preselected_partners = models.TextField() # This field type is a guess. + clarification_request_deadline_date = models.DateField(blank=True, null=True) + population = models.TextField(blank=True, null=True) # This field type is a guess. + + class Meta: + managed = False + db_table = 'project_eoi' + app_label = 'unpp' + + +class ProjectEoiFocalPoints(ReadOnlyModel): + eoi = models.ForeignKey(ProjectEoi, models.PROTECT, related_name='ProjectEoiFocalPoints_eoi') + user = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectEoiFocalPoints_user') + + class Meta: + managed = False + db_table = 'project_eoi_focal_points' + unique_together = (('eoi', 'user'),) + app_label = 'unpp' + + +class ProjectEoiInvitedPartners(ReadOnlyModel): + eoi = models.ForeignKey(ProjectEoi, models.PROTECT, related_name='ProjectEoiInvitedPartners_eoi') + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ProjectEoiInvitedPartners_partner') + + class Meta: + managed = False + db_table = 'project_eoi_invited_partners' + unique_together = (('eoi', 'partner'),) + app_label = 'unpp' + + +class ProjectEoiLocations(ReadOnlyModel): + eoi = models.ForeignKey(ProjectEoi, models.PROTECT, related_name='ProjectEoiLocations_eoi') + point = models.ForeignKey(CommonPoint, models.PROTECT, related_name='ProjectEoiLocations_point') + + class Meta: + managed = False + db_table = 'project_eoi_locations' + unique_together = (('eoi', 'point'),) + app_label = 'unpp' + + +class ProjectEoiReviewers(ReadOnlyModel): + eoi = models.ForeignKey(ProjectEoi, models.PROTECT, related_name='ProjectEoiReviewers_eoi') + user = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectEoiReviewers_user') + + class Meta: + managed = False + db_table = 'project_eoi_reviewers' + unique_together = (('eoi', 'user'),) + app_label = 'unpp' + + +class ProjectEoiSpecializations(ReadOnlyModel): + eoi = models.ForeignKey(ProjectEoi, models.PROTECT, related_name='ProjectEoiSpecializations_eoi') + specialization = models.ForeignKey(CommonSpecialization, models.PROTECT, related_name='ProjectEoiSpecializations_specialization') + + class Meta: + managed = False + db_table = 'project_eoi_specializations' + unique_together = (('eoi', 'specialization'),) + app_label = 'unpp' + + +class ProjectEoiattachment(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + description = models.TextField() + created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectEoiattachment_created_by') + eoi = models.ForeignKey(ProjectEoi, models.PROTECT, related_name='ProjectEoiattachment_eoi') + file = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ProjectEoiattachment_file') + + class Meta: + managed = False + db_table = 'project_eoiattachment' + app_label = 'unpp' + + +class ProjectPin(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + eoi = models.ForeignKey(ProjectEoi, models.PROTECT, related_name='ProjectPin_eoi') + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ProjectPin_partner') + pinned_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectPin_pinned_by') + + class Meta: + managed = False + db_table = 'project_pin' + unique_together = (('eoi', 'partner'),) + app_label = 'unpp' + + +class ReviewPartnerflag(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + flag_type = models.TextField() + is_valid = models.BooleanField(blank=True, null=True) + comment = models.TextField(blank=True, null=True) + contact_person = models.CharField(max_length=255, blank=True, null=True) + contact_phone = models.CharField(max_length=16, blank=True, null=True) + contact_email = models.CharField(max_length=254, blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ReviewPartnerflag_partner') + submitter = models.ForeignKey(AccountUser, models.PROTECT, related_name='ReviewPartnerflag_submitter', blank=True, null=True) + attachment = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ReviewPartnerflag_attachment', blank=True, null=True) + sanctions_match = models.ForeignKey('unpp.SanctionslistSanctionednamematch', models.PROTECT, related_name='ReviewPartnerflag_sanctions_match', blank=True, null=True) + category = models.TextField(blank=True, null=True) + type_history = models.TextField(blank=True, null=True) # This field type is a guess. + validation_comment = models.TextField(blank=True, null=True) + escalation_comment = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'review_partnerflag' + app_label = 'unpp' + + +class ReviewPartnerverification(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + is_valid = models.BooleanField() + is_verified = models.BooleanField() + is_cert_uploaded = models.BooleanField() + cert_uploaded_comment = models.TextField(blank=True, null=True) + is_mm_consistent = models.BooleanField() + mm_consistent_comment = models.TextField(blank=True, null=True) + is_indicate_results = models.BooleanField() + indicate_results_comment = models.TextField(blank=True, null=True) + is_rep_risk = models.BooleanField() + rep_risk_comment = models.TextField(blank=True, null=True) + is_yellow_flag = models.BooleanField() + yellow_flag_comment = models.TextField(blank=True, null=True) + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ReviewPartnerverification_partner') + submitter = models.ForeignKey(AccountUser, models.PROTECT, related_name='ReviewPartnerverification_submitter') + + class Meta: + managed = False + db_table = 'review_partnerverification' + app_label = 'unpp' + + +class SanctionslistSanctioneditem(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + sanctioned_type = models.CharField(max_length=3) + is_active = models.BooleanField() + data_id = models.IntegerField(unique=True) + listed_on = models.DateField(blank=True, null=True) + last_updated = models.DateField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'sanctionslist_sanctioneditem' + app_label = 'unpp' + + +class SanctionslistSanctionedname(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + name = models.TextField() + is_active = models.BooleanField() + item = models.ForeignKey(SanctionslistSanctioneditem, models.PROTECT, related_name='SanctionslistSanctionedname_item') + + class Meta: + managed = False + db_table = 'sanctionslist_sanctionedname' + unique_together = (('item', 'name'),) + app_label = 'unpp' + + +class SanctionslistSanctionednamematch(ReadOnlyModel): + created = models.DateTimeField() + modified = models.DateTimeField() + can_ignore = models.BooleanField() + match_type = models.CharField(max_length=3) + match_text = models.TextField(blank=True, null=True) + can_ignore_text = models.TextField(blank=True, null=True) + name = models.ForeignKey(SanctionslistSanctionedname, models.PROTECT, related_name='SanctionslistSanctionednamematch_name') + partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='SanctionslistSanctionednamematch_partner') + + class Meta: + managed = False + db_table = 'sanctionslist_sanctionednamematch' + unique_together = (('name', 'partner'),) + app_label = 'unpp' + + +class SequencesSequence(ReadOnlyModel): + name = models.CharField(primary_key=True, max_length=100) + last = models.IntegerField() + + class Meta: + managed = False + db_table = 'sequences_sequence' + app_label = 'unpp' diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index c1e1d49fd..aa15f7c1d 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -40,6 +40,7 @@ DATABASE_URL=(str, "postgis://postgres:@127.0.0.1:5432/etools_datamart"), DATABASE_URL_ETOOLS=(str, "postgis://postgres:@127.0.0.1:5432/etools"), DATABASE_URL_PRP=(str, "postgis://postgres:@127.0.0.1:5432/prp"), + DATABASE_URL_UNPP=(str, "postgis://postgres:@127.0.0.1:5432/unpp"), DEBUG=(bool, False), STACK=(str, 'DEVELOPMENT'), API_PAGINATION_OVERRIDE_KEY=(str, 'disable-pagination'), @@ -99,13 +100,15 @@ 'etools': env.db('DATABASE_URL_ETOOLS', engine='etools_datamart.apps.multitenant.postgresql' ), - 'prp': env.db('DATABASE_URL_PRP') + 'prp': env.db('DATABASE_URL_PRP'), + 'unpp': env.db('DATABASE_URL_UNPP'), } # DATABASES['default']['CONN_MAX_AGE'] = 60 DATABASES['etools']['AUTOCOMMIT'] = env('AUTOCOMMIT_EXTERNAL') DATABASES['prp']['AUTOCOMMIT'] = env('AUTOCOMMIT_EXTERNAL') +DATABASES['unpp']['AUTOCOMMIT'] = env('AUTOCOMMIT_EXTERNAL') DATABASE_ROUTERS = [ # 'tenant_schemas.routers.TenantSyncRouter', @@ -115,6 +118,7 @@ # 'django_db_logging'], syncdb=True), router_factory('etools', ['etools'], syncdb=False), router_factory('prp', ['source_prp'], syncdb=False), + router_factory('unpp', ['unpp'], syncdb=False), ] LOGIN_URL = '/login/' @@ -304,10 +308,12 @@ 'etools_datamart.apps.sources.etools', 'etools_datamart.apps.sources.source_prp.apps.Config', + 'etools_datamart.apps.sources.unpp.apps.Config', 'etools_datamart.apps.mart.data', 'etools_datamart.apps.mart.rapidpro', 'etools_datamart.apps.mart.prp', + # 'etools_datamart.apps.mart.unpp', 'etools_datamart.api', 'impersonate', From 0481bdbcd4d3fc26a9e16492d675087b99bf4a29 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Wed, 3 Jun 2020 11:03:15 -0400 Subject: [PATCH 03/12] Move PrpBaseLoader to CommonLoader Allow this loader to be used by UNPP --- src/etools_datamart/apps/etl/loader.py | 111 ++++++++++++++++++++ src/etools_datamart/apps/mart/prp/models.py | 102 +----------------- 2 files changed, 113 insertions(+), 100 deletions(-) diff --git a/src/etools_datamart/apps/etl/loader.py b/src/etools_datamart/apps/etl/loader.py index b3214d678..0307fa233 100644 --- a/src/etools_datamart/apps/etl/loader.py +++ b/src/etools_datamart/apps/etl/loader.py @@ -5,6 +5,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.cache import caches +from django.db import transaction from django.utils import timezone from django.utils.functional import cached_property @@ -22,6 +23,7 @@ RequiredIsRunning, ) from etools_datamart.celery import app +from etools_datamart.sentry import process_exception loadeables = set() locks = caches['lock'] @@ -461,3 +463,112 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor def consistency_check(self): pass + + +class CommonLoader(BaseLoader): + IGNORED_FIELDS = ['source_id', 'id', 'last_modify_date'] + + def get_queryset(self): + if self.config.queryset: + ret = self.config.queryset() + elif self.config.source: + ret = self.config.source.objects.all() + else: # pragma: no cover + raise ValueError( + "Option must define 'queryset' or 'source' attribute" + ) + return ret + + def filter_queryset(self, qs): + use_delta = self.context['only_delta'] and not self.context['is_empty'] + if self.config.filters: + qs = qs.filter(**self.config.filters) + if use_delta and (self.config.last_modify_field and self.last_run): + logger.debug(f"Loader {self}: use deltas") + qs = qs.filter( + **{f"{self.config.last_modify_field}__gte": self.last_run} + ) + return qs + + def load( + self, + *, + verbosity=0, + stdout=None, + ignore_dependencies=False, + max_records=None, + only_delta=True, + run_type=RUN_UNKNOWN, + **kwargs, + ): + self.on_start(run_type) + self.results = EtlResult() + logger.debug(f"Running loader {self}") + lock = self.lock() + truncate = self.config.truncate + try: + if lock: # pragma: no branch + if not ignore_dependencies: + for requirement in self.config.depends: + if requirement.loader.is_running(): + raise RequiredIsRunning(requirement) + if requirement.loader.need_refresh(self): + raise RequiredIsMissing(requirement) + else: + logger.info(f"Loader {requirement} is uptodate") + self.mapping = {} + mart_fields = self.model._meta.concrete_fields + for field in mart_fields: + if field.name not in self.IGNORED_FIELDS: + self.mapping[field.name] = field.name + if self.config.mapping: # pragma: no branch + self.mapping.update(self.config.mapping) + self.update_context(today=timezone.now(), + max_records=max_records, + verbosity=verbosity, + records=0, + only_delta=only_delta, + is_empty=not self.model.objects.exists(), + stdout=stdout) + sid = transaction.savepoint() + try: + self.results.context = self.context + self.fields_to_compare = [ + f for f in self.mapping.keys() if f not in ["seen"] + ] + if truncate: + self.model.objects.truncate() + qs = self.filter_queryset(self.get_queryset()) + for record in qs.all(): + filters = self.config.key(self, record) + values = self.get_values(record) + op = self.process_record(filters, values) + self.increment_counter(op) + + if stdout and verbosity > 0: + stdout.write("\n") + except MaxRecordsException: + pass + except Exception: + transaction.savepoint_rollback(sid) + raise + else: + logger.info(f"Unable to get lock for {self}") + + except (RequiredIsMissing, RequiredIsRunning) as e: + self.on_end(error=e, retry=True) + raise + except BaseException as e: + self.on_end(e) + process_exception(e) + raise + else: + self.on_end(None) + finally: + if lock: # pragma: no branch + try: + lock.release() + except LockError as e: # pragma: no cover + logger.warning(e) + + return self.results diff --git a/src/etools_datamart/apps/mart/prp/models.py b/src/etools_datamart/apps/mart/prp/models.py index 00666590a..2aec48fa0 100644 --- a/src/etools_datamart/apps/mart/prp/models.py +++ b/src/etools_datamart/apps/mart/prp/models.py @@ -51,7 +51,6 @@ from django.db.models import JSONField, Q from django.utils import timezone -from redis.exceptions import LockError from strategy_field.utils import get_attr from etools_datamart.apps.etl.exceptions import MaxRecordsException, RequiredIsMissing, RequiredIsRunning @@ -75,104 +74,7 @@ from .base import PrpDataMartModel -class PrpBaseLoader(BaseLoader): - - def get_queryset(self): - if self.config.queryset: - ret = self.config.queryset() - elif self.config.source: - ret = self.config.source.objects.all() - else: # pragma: no cover - raise ValueError("Option must define 'queryset' or 'source' attribute") - - return ret - - def filter_queryset(self, qs): - use_delta = self.context['only_delta'] and not self.context['is_empty'] - if self.config.filters: - qs = qs.filter(**self.config.filters) - if use_delta and (self.config.last_modify_field and self.last_run): - logger.debug(f"Loader {self}: use deltas") - qs = qs.filter(**{f"{self.config.last_modify_field}__gte": self.last_run}) - return qs - - def load(self, *, verbosity=0, stdout=None, - ignore_dependencies=False, max_records=None, - only_delta=True, run_type=RUN_UNKNOWN, **kwargs): - self.on_start(run_type) - self.results = EtlResult() - logger.debug(f"Running loader {self}") - lock = self.lock() - truncate = self.config.truncate - try: - if lock: # pragma: no branch - if not ignore_dependencies: - for requirement in self.config.depends: - if requirement.loader.is_running(): - raise RequiredIsRunning(requirement) - if requirement.loader.need_refresh(self): - raise RequiredIsMissing(requirement) - else: - logger.info(f"Loader {requirement} is uptodate") - self.mapping = {} - mart_fields = self.model._meta.concrete_fields - for field in mart_fields: - if field.name not in ['source_id', 'id', 'last_modify_date']: - self.mapping[field.name] = field.name - if self.config.mapping: # pragma: no branch - self.mapping.update(self.config.mapping) - self.update_context(today=timezone.now(), - max_records=max_records, - verbosity=verbosity, - records=0, - only_delta=only_delta, - is_empty=not self.model.objects.exists(), - stdout=stdout) - sid = transaction.savepoint() - try: - self.results.context = self.context - self.fields_to_compare = [f for f in self.mapping.keys() if f not in ["seen"]] - if truncate: - self.model.objects.truncate() - qs = self.filter_queryset(self.get_queryset()) - for record in qs.all(): - filters = self.config.key(self, record) - values = self.get_values(record) - op = self.process_record(filters, values) - self.increment_counter(op) - - if stdout and verbosity > 0: - stdout.write("\n") - # deleted = self.model.objects.exclude(seen=today).delete()[0] - # self.results.deleted = deleted - except MaxRecordsException: - pass - except Exception: - transaction.savepoint_rollback(sid) - raise - else: - logger.info(f"Unable to get lock for {self}") - - except (RequiredIsMissing, RequiredIsRunning) as e: - self.on_end(error=e, retry=True) - raise - except BaseException as e: - self.on_end(e) - process_exception(e) - raise - else: - self.on_end(None) - finally: - if lock: # pragma: no branch - try: - lock.release() - except LockError as e: # pragma: no cover - logger.warning(e) - - return self.results - - -class IndicatorByLocationLoader(PrpBaseLoader): +class IndicatorByLocationLoader(CommonLoader): def get_location_levelname(self, record, values, field_name): pass @@ -229,7 +131,7 @@ class Options: } -class DataReportLoader(PrpBaseLoader): +class DataReportLoader(CommonLoader): def get_queryset(self): # all_progress_reports = UnicefProgressreport.objects.all() From 8ee7c7f6c7e9fc2935bd8bdae05667ea0370cee7 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 4 Jun 2020 10:24:30 -0400 Subject: [PATCH 04/12] Update label for unpp sources --- src/etools_datamart/apps/sources/unpp/apps.py | 2 +- .../apps/sources/unpp/models.py | 148 +++++++++--------- src/etools_datamart/config/settings.py | 2 +- 3 files changed, 76 insertions(+), 76 deletions(-) diff --git a/src/etools_datamart/apps/sources/unpp/apps.py b/src/etools_datamart/apps/sources/unpp/apps.py index ae9a0e722..acf0a47ec 100644 --- a/src/etools_datamart/apps/sources/unpp/apps.py +++ b/src/etools_datamart/apps/sources/unpp/apps.py @@ -4,7 +4,7 @@ class Config(AppConfig): name = 'etools_datamart.apps.sources.unpp' verbose_name = "UNPP" - label = 'unpp' + label = 'source_unpp' def ready(self): pass diff --git a/src/etools_datamart/apps/sources/unpp/models.py b/src/etools_datamart/apps/sources/unpp/models.py index 919f4fef2..f374829f4 100644 --- a/src/etools_datamart/apps/sources/unpp/models.py +++ b/src/etools_datamart/apps/sources/unpp/models.py @@ -19,18 +19,18 @@ class AccountUser(ReadOnlyModel): class Meta: managed = False db_table = 'account_user' - app_label = 'unpp' + app_label = 'source_unpp' class AccountUserGroups(ReadOnlyModel): user = models.ForeignKey(AccountUser, models.PROTECT, related_name='AccountUserGroups_user') - group = models.ForeignKey('unpp.AuthGroup', models.PROTECT, related_name='AccountUserGroups_group') + group = models.ForeignKey('source_unpp.AuthGroup', models.PROTECT, related_name='AccountUserGroups_group') class Meta: managed = False db_table = 'account_user_groups' unique_together = (('user', 'group'),) - app_label = 'unpp' + app_label = 'source_unpp' class AccountUserprofile(ReadOnlyModel): @@ -43,7 +43,7 @@ class AccountUserprofile(ReadOnlyModel): class Meta: managed = False db_table = 'account_userprofile' - app_label = 'unpp' + app_label = 'source_unpp' class AgencyAgency(ReadOnlyModel): @@ -52,13 +52,13 @@ class AgencyAgency(ReadOnlyModel): class Meta: managed = False db_table = 'agency_agency' - app_label = 'unpp' + app_label = 'source_unpp' class AgencyAgencymember(ReadOnlyModel): created = models.DateTimeField() modified = models.DateTimeField() - office = models.ForeignKey('unpp.AgencyAgencyoffice', models.PROTECT, related_name='AgencyAgencymember_office') + office = models.ForeignKey('source_unpp.AgencyAgencyoffice', models.PROTECT, related_name='AgencyAgencymember_office') user = models.ForeignKey(AccountUser, models.PROTECT, related_name='AgencyAgencymember_user') telephone = models.CharField(max_length=255, blank=True, null=True) role = models.TextField() @@ -67,7 +67,7 @@ class Meta: managed = False db_table = 'agency_agencymember' unique_together = (('user', 'office'),) - app_label = 'unpp' + app_label = 'source_unpp' class AgencyAgencyoffice(ReadOnlyModel): @@ -80,7 +80,7 @@ class Meta: managed = False db_table = 'agency_agencyoffice' unique_together = (('agency', 'country'),) - app_label = 'unpp' + app_label = 'source_unpp' class AgencyAgencyprofile(ReadOnlyModel): @@ -92,7 +92,7 @@ class AgencyAgencyprofile(ReadOnlyModel): class Meta: managed = False db_table = 'agency_agencyprofile' - app_label = 'unpp' + app_label = 'source_unpp' class AgencyOtheragency(ReadOnlyModel): @@ -103,7 +103,7 @@ class AgencyOtheragency(ReadOnlyModel): class Meta: managed = False db_table = 'agency_otheragency' - app_label = 'unpp' + app_label = 'source_unpp' class AuthGroup(ReadOnlyModel): @@ -112,7 +112,7 @@ class AuthGroup(ReadOnlyModel): class Meta: managed = False db_table = 'auth_group' - app_label = 'unpp' + app_label = 'source_unpp' class CommonAdminlevel1(ReadOnlyModel): @@ -123,7 +123,7 @@ class Meta: managed = False db_table = 'common_adminlevel1' unique_together = (('name', 'country_code'),) - app_label = 'unpp' + app_label = 'source_unpp' class CommonCommonfile(ReadOnlyModel): @@ -134,7 +134,7 @@ class CommonCommonfile(ReadOnlyModel): class Meta: managed = False db_table = 'common_commonfile' - app_label = 'unpp' + app_label = 'source_unpp' class CommonPoint(ReadOnlyModel): @@ -145,7 +145,7 @@ class CommonPoint(ReadOnlyModel): class Meta: managed = False db_table = 'common_point' - app_label = 'unpp' + app_label = 'source_unpp' class CommonSector(ReadOnlyModel): @@ -154,7 +154,7 @@ class CommonSector(ReadOnlyModel): class Meta: managed = False db_table = 'common_sector' - app_label = 'unpp' + app_label = 'source_unpp' class CommonSpecialization(ReadOnlyModel): @@ -164,7 +164,7 @@ class CommonSpecialization(ReadOnlyModel): class Meta: managed = False db_table = 'common_specialization' - app_label = 'unpp' + app_label = 'source_unpp' class DjangoContentType(ReadOnlyModel): @@ -175,7 +175,7 @@ class Meta: managed = False db_table = 'django_content_type' unique_together = (('app_label', 'model'),) - app_label = 'unpp' + app_label = 'source_unpp' class DjangoSite(ReadOnlyModel): @@ -185,7 +185,7 @@ class DjangoSite(ReadOnlyModel): class Meta: managed = False db_table = 'django_site' - app_label = 'unpp' + app_label = 'source_unpp' class ExternalsPartnervendornumber(ReadOnlyModel): @@ -193,14 +193,14 @@ class ExternalsPartnervendornumber(ReadOnlyModel): modified = models.DateTimeField() number = models.TextField() agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='ExternalsPartnervendornumber_agency') - partner = models.ForeignKey('unpp.PartnerPartner', models.PROTECT, related_name='ExternalsPartnervendornumber_partner') + partner = models.ForeignKey('source_unpp.PartnerPartner', models.PROTECT, related_name='ExternalsPartnervendornumber_partner') business_area = models.TextField(blank=True, null=True) class Meta: managed = False db_table = 'externals_partnervendornumber' unique_together = (('agency', 'partner', 'business_area', 'number'),) - app_label = 'unpp' + app_label = 'source_unpp' class ExternalsUnicefvendordata(ReadOnlyModel): @@ -217,7 +217,7 @@ class Meta: managed = False db_table = 'externals_unicefvendordata' unique_together = (('business_area', 'vendor_number', 'year'),) - app_label = 'unpp' + app_label = 'source_unpp' class NotificationNotification(ReadOnlyModel): @@ -232,7 +232,7 @@ class NotificationNotification(ReadOnlyModel): class Meta: managed = False db_table = 'notification_notification' - app_label = 'unpp' + app_label = 'source_unpp' class NotificationNotifieduser(ReadOnlyModel): @@ -247,7 +247,7 @@ class Meta: managed = False db_table = 'notification_notifieduser' unique_together = (('notification', 'recipient'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartner(ReadOnlyModel): @@ -275,7 +275,7 @@ class Meta: managed = False db_table = 'partner_partner' unique_together = (('legal_name', 'country_code', 'hq'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerLocationFieldOffices(ReadOnlyModel): @@ -286,7 +286,7 @@ class Meta: managed = False db_table = 'partner_partner_location_field_offices' unique_together = (('partner', 'point'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerauditassessment(ReadOnlyModel): @@ -302,7 +302,7 @@ class PartnerPartnerauditassessment(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerauditassessment' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerauditreport(ReadOnlyModel): @@ -317,7 +317,7 @@ class PartnerPartnerauditreport(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerauditreport' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerauthorisedofficer(ReadOnlyModel): @@ -335,7 +335,7 @@ class PartnerPartnerauthorisedofficer(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerauthorisedofficer' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerbudget(ReadOnlyModel): @@ -350,7 +350,7 @@ class Meta: managed = False db_table = 'partner_partnerbudget' unique_together = (('partner', 'year'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnercapacityassessment(ReadOnlyModel): @@ -365,7 +365,7 @@ class PartnerPartnercapacityassessment(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnercapacityassessment' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnercollaborationevidence(ReadOnlyModel): @@ -382,7 +382,7 @@ class PartnerPartnercollaborationevidence(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnercollaborationevidence' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnercollaborationpartnership(ReadOnlyModel): @@ -398,7 +398,7 @@ class Meta: managed = False db_table = 'partner_partnercollaborationpartnership' unique_together = (('partner', 'agency'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerdirector(ReadOnlyModel): @@ -417,7 +417,7 @@ class PartnerPartnerdirector(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerdirector' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerexperience(ReadOnlyModel): @@ -431,7 +431,7 @@ class PartnerPartnerexperience(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerexperience' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerfunding(ReadOnlyModel): @@ -445,7 +445,7 @@ class PartnerPartnerfunding(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerfunding' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnergoverningdocument(ReadOnlyModel): @@ -453,13 +453,13 @@ class PartnerPartnergoverningdocument(ReadOnlyModel): modified = models.DateTimeField() editable = models.BooleanField() document = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='PartnerPartnergoverningdocument_document') - profile = models.ForeignKey('unpp.PartnerPartnerprofile', models.PROTECT, related_name='PartnerPartnergoverningdocument_profile') + profile = models.ForeignKey('source_unpp.PartnerPartnerprofile', models.PROTECT, related_name='PartnerPartnergoverningdocument_profile') created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnergoverningdocument_created_by') class Meta: managed = False db_table = 'partner_partnergoverningdocument' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerheadorganization(ReadOnlyModel): @@ -479,7 +479,7 @@ class PartnerPartnerheadorganization(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerheadorganization' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerinternalcontrol(ReadOnlyModel): @@ -494,7 +494,7 @@ class Meta: managed = False db_table = 'partner_partnerinternalcontrol' unique_together = (('partner', 'functional_responsibility'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnermailingaddress(ReadOnlyModel): @@ -515,7 +515,7 @@ class PartnerPartnermailingaddress(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnermailingaddress' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnermandatemission(ReadOnlyModel): @@ -542,7 +542,7 @@ class PartnerPartnermandatemission(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnermandatemission' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnermember(ReadOnlyModel): @@ -557,7 +557,7 @@ class Meta: managed = False db_table = 'partner_partnermember' unique_together = (('user', 'partner'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerotherinfo(ReadOnlyModel): @@ -575,7 +575,7 @@ class PartnerPartnerotherinfo(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerotherinfo' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerpolicyarea(ReadOnlyModel): @@ -590,7 +590,7 @@ class Meta: managed = False db_table = 'partner_partnerpolicyarea' unique_together = (('partner', 'area'),) - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerprofile(ReadOnlyModel): @@ -636,7 +636,7 @@ class PartnerPartnerprofile(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerprofile' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerregistrationdocument(ReadOnlyModel): @@ -654,7 +654,7 @@ class PartnerPartnerregistrationdocument(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerregistrationdocument' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerreporting(ReadOnlyModel): @@ -670,7 +670,7 @@ class PartnerPartnerreporting(ReadOnlyModel): class Meta: managed = False db_table = 'partner_partnerreporting' - app_label = 'unpp' + app_label = 'source_unpp' class PartnerPartnerreview(ReadOnlyModel): @@ -684,14 +684,14 @@ class PartnerPartnerreview(ReadOnlyModel): does_recommend = models.BooleanField(blank=True, null=True) comment = models.TextField() agency = models.ForeignKey(AgencyAgency, models.PROTECT, related_name='PartnerPartnerreview_agency') - eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='PartnerPartnerreview_eoi') + eoi = models.ForeignKey('source_unpp.ProjectEoi', models.PROTECT, related_name='PartnerPartnerreview_eoi') partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='PartnerPartnerreview_partner') reviewer = models.ForeignKey(AccountUser, models.PROTECT, related_name='PartnerPartnerreview_reviewer') class Meta: managed = False db_table = 'partner_partnerreview' - app_label = 'unpp' + app_label = 'source_unpp' class ProjectApplication(ReadOnlyModel): @@ -703,7 +703,7 @@ class ProjectApplication(ReadOnlyModel): did_accept = models.BooleanField() ds_justification_select = models.TextField(blank=True, null=True) # This field type is a guess. justification_reason = models.TextField(blank=True, null=True) - eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='ProjectApplication_eoi', blank=True, null=True) + eoi = models.ForeignKey('source_unpp.ProjectEoi', models.PROTECT, related_name='ProjectApplication_eoi', blank=True, null=True) partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ProjectApplication_partner') submitter = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectApplication_submitter') proposal_of_eoi_details = models.TextField() # This field type is a guess. @@ -711,7 +711,7 @@ class ProjectApplication(ReadOnlyModel): did_withdraw = models.BooleanField() withdraw_reason = models.TextField(blank=True, null=True) did_decline = models.BooleanField() - eoi_converted = models.OneToOneField('unpp.ProjectEoi', models.PROTECT, related_name='ProjectApplication_eoi_converted', blank=True, null=True) + eoi_converted = models.OneToOneField('source_unpp.ProjectEoi', models.PROTECT, related_name='ProjectApplication_eoi_converted', blank=True, null=True) cn = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ProjectApplication_cn', blank=True, null=True) accept_notification = models.OneToOneField(NotificationNotification, models.PROTECT, related_name='ProjectApplication_accept_notification', blank=True, null=True) partner_decision_date = models.DateField(blank=True, null=True) @@ -726,7 +726,7 @@ class Meta: managed = False db_table = 'project_application' unique_together = (('eoi', 'partner'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectApplicationLocationsProposalOfEoi(ReadOnlyModel): @@ -737,7 +737,7 @@ class Meta: managed = False db_table = 'project_application_locations_proposal_of_eoi' unique_together = (('application', 'point'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectApplicationfeedback(ReadOnlyModel): @@ -750,7 +750,7 @@ class ProjectApplicationfeedback(ReadOnlyModel): class Meta: managed = False db_table = 'project_applicationfeedback' - app_label = 'unpp' + app_label = 'source_unpp' class ProjectAssessment(ReadOnlyModel): @@ -772,7 +772,7 @@ class Meta: managed = False db_table = 'project_assessment' unique_together = (('reviewer', 'application'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectClarificationrequestanswerfile(ReadOnlyModel): @@ -780,13 +780,13 @@ class ProjectClarificationrequestanswerfile(ReadOnlyModel): modified = models.DateTimeField() title = models.TextField() created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectClarificationrequestanswerfile_created_by') - eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='ProjectClarificationrequestanswerfile_eoi') + eoi = models.ForeignKey('source_unpp.ProjectEoi', models.PROTECT, related_name='ProjectClarificationrequestanswerfile_eoi') file = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ProjectClarificationrequestanswerfile_file') class Meta: managed = False db_table = 'project_clarificationrequestanswerfile' - app_label = 'unpp' + app_label = 'source_unpp' class ProjectClarificationrequestquestion(ReadOnlyModel): @@ -794,13 +794,13 @@ class ProjectClarificationrequestquestion(ReadOnlyModel): modified = models.DateTimeField() question = models.TextField() created_by = models.ForeignKey(AccountUser, models.PROTECT, related_name='ProjectClarificationrequestquestion_created_by') - eoi = models.ForeignKey('unpp.ProjectEoi', models.PROTECT, related_name='ProjectClarificationrequestquestion_eoi') + eoi = models.ForeignKey('source_unpp.ProjectEoi', models.PROTECT, related_name='ProjectClarificationrequestquestion_eoi') partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ProjectClarificationrequestquestion_partner') class Meta: managed = False db_table = 'project_clarificationrequestquestion' - app_label = 'unpp' + app_label = 'source_unpp' class ProjectEoi(ReadOnlyModel): @@ -842,7 +842,7 @@ class ProjectEoi(ReadOnlyModel): class Meta: managed = False db_table = 'project_eoi' - app_label = 'unpp' + app_label = 'source_unpp' class ProjectEoiFocalPoints(ReadOnlyModel): @@ -853,7 +853,7 @@ class Meta: managed = False db_table = 'project_eoi_focal_points' unique_together = (('eoi', 'user'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectEoiInvitedPartners(ReadOnlyModel): @@ -864,7 +864,7 @@ class Meta: managed = False db_table = 'project_eoi_invited_partners' unique_together = (('eoi', 'partner'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectEoiLocations(ReadOnlyModel): @@ -875,7 +875,7 @@ class Meta: managed = False db_table = 'project_eoi_locations' unique_together = (('eoi', 'point'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectEoiReviewers(ReadOnlyModel): @@ -886,7 +886,7 @@ class Meta: managed = False db_table = 'project_eoi_reviewers' unique_together = (('eoi', 'user'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectEoiSpecializations(ReadOnlyModel): @@ -897,7 +897,7 @@ class Meta: managed = False db_table = 'project_eoi_specializations' unique_together = (('eoi', 'specialization'),) - app_label = 'unpp' + app_label = 'source_unpp' class ProjectEoiattachment(ReadOnlyModel): @@ -911,7 +911,7 @@ class ProjectEoiattachment(ReadOnlyModel): class Meta: managed = False db_table = 'project_eoiattachment' - app_label = 'unpp' + app_label = 'source_unpp' class ProjectPin(ReadOnlyModel): @@ -925,7 +925,7 @@ class Meta: managed = False db_table = 'project_pin' unique_together = (('eoi', 'partner'),) - app_label = 'unpp' + app_label = 'source_unpp' class ReviewPartnerflag(ReadOnlyModel): @@ -940,7 +940,7 @@ class ReviewPartnerflag(ReadOnlyModel): partner = models.ForeignKey(PartnerPartner, models.PROTECT, related_name='ReviewPartnerflag_partner') submitter = models.ForeignKey(AccountUser, models.PROTECT, related_name='ReviewPartnerflag_submitter', blank=True, null=True) attachment = models.ForeignKey(CommonCommonfile, models.PROTECT, related_name='ReviewPartnerflag_attachment', blank=True, null=True) - sanctions_match = models.ForeignKey('unpp.SanctionslistSanctionednamematch', models.PROTECT, related_name='ReviewPartnerflag_sanctions_match', blank=True, null=True) + sanctions_match = models.ForeignKey('source_unpp.SanctionslistSanctionednamematch', models.PROTECT, related_name='ReviewPartnerflag_sanctions_match', blank=True, null=True) category = models.TextField(blank=True, null=True) type_history = models.TextField(blank=True, null=True) # This field type is a guess. validation_comment = models.TextField(blank=True, null=True) @@ -949,7 +949,7 @@ class ReviewPartnerflag(ReadOnlyModel): class Meta: managed = False db_table = 'review_partnerflag' - app_label = 'unpp' + app_label = 'source_unpp' class ReviewPartnerverification(ReadOnlyModel): @@ -973,7 +973,7 @@ class ReviewPartnerverification(ReadOnlyModel): class Meta: managed = False db_table = 'review_partnerverification' - app_label = 'unpp' + app_label = 'source_unpp' class SanctionslistSanctioneditem(ReadOnlyModel): @@ -988,7 +988,7 @@ class SanctionslistSanctioneditem(ReadOnlyModel): class Meta: managed = False db_table = 'sanctionslist_sanctioneditem' - app_label = 'unpp' + app_label = 'source_unpp' class SanctionslistSanctionedname(ReadOnlyModel): @@ -1002,7 +1002,7 @@ class Meta: managed = False db_table = 'sanctionslist_sanctionedname' unique_together = (('item', 'name'),) - app_label = 'unpp' + app_label = 'source_unpp' class SanctionslistSanctionednamematch(ReadOnlyModel): @@ -1019,7 +1019,7 @@ class Meta: managed = False db_table = 'sanctionslist_sanctionednamematch' unique_together = (('name', 'partner'),) - app_label = 'unpp' + app_label = 'source_unpp' class SequencesSequence(ReadOnlyModel): @@ -1029,4 +1029,4 @@ class SequencesSequence(ReadOnlyModel): class Meta: managed = False db_table = 'sequences_sequence' - app_label = 'unpp' + app_label = 'source_unpp' diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index aa15f7c1d..f27f658ad 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -118,7 +118,7 @@ # 'django_db_logging'], syncdb=True), router_factory('etools', ['etools'], syncdb=False), router_factory('prp', ['source_prp'], syncdb=False), - router_factory('unpp', ['unpp'], syncdb=False), + router_factory('unpp', ['source_unpp'], syncdb=False), ] LOGIN_URL = '/login/' From 7f8f0312d616291ad4e724355534dcd24d9b98f0 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 4 Jun 2020 10:25:10 -0400 Subject: [PATCH 05/12] Add initial unpp loaders --- .../apps/mart/unpp/__init__.py | 0 src/etools_datamart/apps/mart/unpp/admin.py | 24 +++ src/etools_datamart/apps/mart/unpp/base.py | 40 +++++ .../apps/mart/unpp/migrations/0001_initial.py | 52 +++++++ .../apps/mart/unpp/migrations/__init__.py | 0 src/etools_datamart/apps/mart/unpp/models.py | 137 ++++++++++++++++++ src/etools_datamart/config/settings.py | 2 +- 7 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 src/etools_datamart/apps/mart/unpp/__init__.py create mode 100644 src/etools_datamart/apps/mart/unpp/admin.py create mode 100644 src/etools_datamart/apps/mart/unpp/base.py create mode 100644 src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py create mode 100644 src/etools_datamart/apps/mart/unpp/migrations/__init__.py create mode 100644 src/etools_datamart/apps/mart/unpp/models.py diff --git a/src/etools_datamart/apps/mart/unpp/__init__.py b/src/etools_datamart/apps/mart/unpp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/mart/unpp/admin.py b/src/etools_datamart/apps/mart/unpp/admin.py new file mode 100644 index 000000000..9b88d0cdc --- /dev/null +++ b/src/etools_datamart/apps/mart/unpp/admin.py @@ -0,0 +1,24 @@ +import logging + +from django.contrib.admin import ModelAdmin, register + +from etools_datamart.apps.mart.unpp import models +from etools_datamart.libs.truncate import TruncateTableMixin + +logger = logging.getLogger(__name__) + + +@register(models.Location) +class LocationAdmin(TruncateTableMixin, ModelAdmin): + list_display = ( + 'source_id', + 'name', + 'country_code', + 'latitude', + 'longitude', + ) + + +@register(models.Application) +class ApplicationAdmin(TruncateTableMixin, ModelAdmin): + list_display = ('__str__',) diff --git a/src/etools_datamart/apps/mart/unpp/base.py b/src/etools_datamart/apps/mart/unpp/base.py new file mode 100644 index 000000000..ad7105576 --- /dev/null +++ b/src/etools_datamart/apps/mart/unpp/base.py @@ -0,0 +1,40 @@ +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from celery.local import class_property + +from etools_datamart.apps.core.models import DataMartManager +from etools_datamart.apps.etl.base import DataMartModelBase +from etools_datamart.apps.etl.loader import BaseLoaderOptions + + +class UNPPBaseManager(DataMartManager): + pass + + +class UNPPDataMartModelBase(DataMartModelBase): + loader_option_class = BaseLoaderOptions + loader_class = DataMartModelBase + + +class UNPPDataMartModel(models.Model, metaclass=UNPPDataMartModelBase): + source_id = models.IntegerField(blank=True, null=True, db_index=True) + last_modify_date = models.DateTimeField(blank=True, auto_now=True) + seen = models.DateTimeField(blank=True, null=True) + + objects = UNPPBaseManager() + + class Meta: + abstract = True + + @class_property + def service(self): + from unicef_rest_framework.models import Service + return Service.objects.get( + source_model=ContentType.objects.get_for_model(self), + ) + + @class_property + def linked_services(self): + from unicef_rest_framework.models import Service + return [s for s in Service.objects.all() if s.managed_model == self] diff --git a/src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py b/src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py new file mode 100644 index 000000000..2401e38b3 --- /dev/null +++ b/src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py @@ -0,0 +1,52 @@ +# Generated by Django 2.2.11 on 2020-06-03 17:43 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name='Location', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source_id', models.IntegerField(blank=True, db_index=True, null=True)), + ('last_modify_date', models.DateTimeField(auto_now=True)), + ('seen', models.DateTimeField(blank=True, null=True)), + ('latitude', models.FloatField(blank=True, null=True)), + ('longitude', models.FloatField(blank=True, null=True)), + ('name', models.CharField(blank=True, max_length=255, null=True)), + ('country_code', models.CharField(blank=True, max_length=3, null=True)), + ], + ), + migrations.CreateModel( + name='Application', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source_id', models.IntegerField(blank=True, db_index=True, null=True)), + ('last_modify_date', models.DateTimeField(auto_now=True)), + ('seen', models.DateTimeField(blank=True, null=True)), + ('type_of_call', models.CharField(blank=True, max_length=254, null=True)), + ('title', models.CharField(blank=True, max_length=254, null=True)), + ('agency', models.CharField(blank=True, max_length=254, null=True)), + ('created_by', models.CharField(blank=True, max_length=512, null=True)), + ('focal_points', models.TextField(blank=True, null=True)), + ('focal_points_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('agency_office', models.CharField(blank=True, max_length=254, null=True)), + ('cn_template', models.CharField(blank=True, max_length=100, null=True)), + ('specializations', models.TextField(blank=True, null=True)), + ('specializations_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('description', models.TextField(blank=True, null=True)), + ('goal', models.TextField(blank=True, null=True)), + ('other_information', models.TextField(blank=True, null=True)), + ('estimated_start_date', models.DateField(blank=True, null=True)), + ('locations', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='unpp.Location')), + ], + ), + ] diff --git a/src/etools_datamart/apps/mart/unpp/migrations/__init__.py b/src/etools_datamart/apps/mart/unpp/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/mart/unpp/models.py b/src/etools_datamart/apps/mart/unpp/models.py new file mode 100644 index 000000000..30c3cefee --- /dev/null +++ b/src/etools_datamart/apps/mart/unpp/models.py @@ -0,0 +1,137 @@ +from django.contrib.postgres.fields import JSONField +from django.db import models + +from etools_datamart.apps.etl.loader import CommonLoader +from etools_datamart.apps.mart.unpp.base import UNPPDataMartModel +from etools_datamart.apps.sources.unpp.models import CommonPoint, ProjectApplication + + +class Location(UNPPDataMartModel): + latitude = models.FloatField(blank=True, null=True) + longitude = models.FloatField(blank=True, null=True) + name = models.CharField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=3, blank=True, null=True) + + loader = CommonLoader() + + class Meta: + app_label = 'unpp' + + class Options: + key = lambda loader, record: {'source_id': record.pk} + source = CommonPoint + mapping = { + "latitude": "latitude", + "longitude": "longitude", + "name": "admin_level_1.name", + "country_code": "admin_level_1.country_code", + } + + +class ApplicationLoader(CommonLoader): + def get_queryset(self): + qs = ProjectApplication.objects.select_related( + "eoi" + ).prefetch_related( + "focal_points", + ) + return qs + + def get_focal_points( + self, + record: ProjectApplication, + values: dict, + **kwargs, + ): + data = [] + ret = [] + for member in record.focal_points.all(): + # member is AccountUser + ret.append( + "{0.last_name} {0.first_name} ({0.email}) {0.phone}".format( + member, + ) + ) + data.append(dict( + last_name=member.last_name, + first_name=member.first_name, + email=member.email, + phone=member.phone, + )) + + values['focal_points_data'] = data + return ", ".join(ret) + + def get_specializations( + self, + record: ProjectApplication, + values: dict, + **kwargs, + ): + data = [] + ret = [] + if record.eoi: + for specialization in record.eoi.specializations.all(): + # member is ProjectEoiSpecializations + ret.append( + "{0.name} ({0.category})".format( + specialization.specialization, + ) + ) + data.append(dict( + name=specialization.specialization.name, + category=specialization.specialization.category, + )) + + values['specializations_data'] = data + return ", ".join(ret) + + +class Application(UNPPDataMartModel): + type_of_call = models.CharField(max_length=254, null=True, blank=True) + title = models.CharField(max_length=254, null=True, blank=True) + agency = models.CharField(max_length=254, null=True, blank=True) + created_by = models.CharField(max_length=512, blank=True, null=True) + focal_points = models.TextField(blank=True, null=True) + focal_points_data = JSONField(blank=True, null=True, default=dict) + locations = models.ForeignKey( + "Location", + models.DO_NOTHING, + blank=True, + null=True, + ) + agency_office = models.CharField(max_length=254, null=True, blank=True) + cn_template = models.CharField(max_length=100, blank=True, null=True) + specializations = models.TextField(blank=True, null=True) + specializations_data = JSONField(blank=True, null=True, default=dict) + description = models.TextField(blank=True, null=True) + goal = models.TextField(blank=True, null=True) + other_information = models.TextField(blank=True, null=True) + estimated_start_date = models.DateField(blank=True, null=True) + + loader = ApplicationLoader() + + class Meta: + app_label = 'unpp' + + class Options: + depends = (Location,) + key = lambda loader, record: {'source_id': record.pk} + source = ProjectApplication + mapping = { + "type_of_call": "eoi.display_type", # TODO get actual CFEI_TYPES value + "title": "eoi.title", + "agency": "agency.name", + "created_by": "submitter.fullname", + "focal_points": "-", + "focal_points_data": "i", + "location": Location, + "agency_office": "eoi.agency_office.country", + "cn_template": "eoi.cn_template", + "specializations": "-", + "specializations_data": "i", + "description": "description", + "goal": "goal", + "other_information": "other_information", + "start_date": "start_date", + } diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index f27f658ad..72e80f914 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -313,7 +313,7 @@ 'etools_datamart.apps.mart.data', 'etools_datamart.apps.mart.rapidpro', 'etools_datamart.apps.mart.prp', - # 'etools_datamart.apps.mart.unpp', + 'etools_datamart.apps.mart.unpp', 'etools_datamart.api', 'impersonate', From 97fe6b80b20011af96646b57edc56f79c7836fb1 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 4 Jun 2020 12:13:10 -0400 Subject: [PATCH 06/12] Add unpp to testing --- tests/.coveragerc | 2 + .../test_utilities/factories/__init__.py | 1 + .../test_utilities/factories/unpp.py | 13 +++ tests/api/interfaces/unpp.py | 94 +++++++++++++++++++ tests/etl/test_etl_loaders.py | 2 + 5 files changed, 112 insertions(+) create mode 100644 tests/_test_lib/test_utilities/factories/unpp.py create mode 100644 tests/api/interfaces/unpp.py diff --git a/tests/.coveragerc b/tests/.coveragerc index 9d89cb06f..ceba14d32 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -20,6 +20,8 @@ omit = **/tracking/management/commands/track.py **/data/management/commands/sqlall.py **/source_prp/management/commands/inspectprp.py + **/etools_datamart/apps/prp/management/commands/inspectprp.py + **/etools_datamart/apps/unpp/management/commands/inspectunpp.py tests/** **/migrations/** **/custom_migrations/** diff --git a/tests/_test_lib/test_utilities/factories/__init__.py b/tests/_test_lib/test_utilities/factories/__init__.py index a0f996dcd..b127fba27 100644 --- a/tests/_test_lib/test_utilities/factories/__init__.py +++ b/tests/_test_lib/test_utilities/factories/__init__.py @@ -2,4 +2,5 @@ from .data import * # noqa from .etools import * # noqa from .prp import * # noqa +from .unpp import * # noqa from .rapidpro import * # noqa diff --git a/tests/_test_lib/test_utilities/factories/unpp.py b/tests/_test_lib/test_utilities/factories/unpp.py new file mode 100644 index 000000000..5a37b6684 --- /dev/null +++ b/tests/_test_lib/test_utilities/factories/unpp.py @@ -0,0 +1,13 @@ +from test_utilities.factories import RegisterModelFactory + +from etools_datamart.apps.mart.unpp import models + + +class LocationMartModelFactory(RegisterModelFactory): + class Meta: + model = models.Location + + +class ApplicationMartModelFactory(RegisterModelFactory): + class Meta: + model = models.Application diff --git a/tests/api/interfaces/unpp.py b/tests/api/interfaces/unpp.py new file mode 100644 index 000000000..044cbab10 --- /dev/null +++ b/tests/api/interfaces/unpp.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +import os +from functools import wraps + +import pytest +from drf_api_checker.pytest import contract +from drf_api_checker.recorder import BASE_DATADIR, Recorder +from rest_framework.test import APIClient +from test_utilities.factories import factories_registry, UserFactory + +from etools_datamart.api.urls import router + + +class MyRecorder(Recorder): + @property + def client(self): + user = UserFactory(is_superuser=True) + client = APIClient() + client.force_authenticate(user) + return client + + +def frozenfixture2(use_request=False): + def deco(func): + from drf_api_checker.utils import load_fixtures, dump_fixtures + from drf_api_checker.fs import mktree + + @wraps(func) + def _inner(*args, **kwargs): + parts = [os.path.dirname(func.__code__.co_filename), + BASE_DATADIR, + func.__module__, + func.__name__, ] + if 'request' in kwargs: + request = kwargs['request'] + viewset = request.getfixturevalue('viewset') + parts.append(viewset.__name__) + + destination = os.path.join(*parts + ) + '.fixture.json' + if os.path.exists(destination) and not os.environ.get('API_CHECKER_RESET'): + return load_fixtures(destination)[func.__name__] + mktree(os.path.dirname(destination)) + data = func(*args, **kwargs) + dump_fixtures({func.__name__: data}, destination) + return data + + return pytest.fixture(_inner) + + return deco + + +def pytest_generate_tests(metafunc, *args): + if 'viewset' in metafunc.fixturenames: + params = [] + ids = [] + for prefix, viewset, basenametry in router.registry: + if prefix.startswith('sources/unpp/'): + sers = viewset.serializers_fieldsets.keys() + for ser in sers: + params.append([viewset, ser]) + ids.append(f'{prefix}/{viewset.__name__}-{ser}') + metafunc.parametrize("viewset,serializer", params, ids=ids) + + +@frozenfixture2() +def data(db, request): + # TIPS: database access is forbidden in pytest_generate_tests + viewset = request.getfixturevalue('viewset') + factory = factories_registry[viewset.serializer_class.Meta.model] + data = (factory(schema_name='bolivia'), + factory(schema_name='chad'), + factory(schema_name='lebanon')) + return data + + +@contract(recorder_class=MyRecorder) +def test_list(viewset, serializer, data): + url = f"{viewset.get_service().endpoint}" + return [url, {'-serializer': serializer}] + + +@frozenfixture2() +def record(db, request): + # TIPS: database access is forbidden in pytest_generate_tests + viewset = request.getfixturevalue('viewset') + factory = factories_registry[viewset.serializer_class.Meta.model] + return factory(schema_name='bolivia') + + +@contract(recorder_class=MyRecorder) +def test_record(viewset, serializer, record): + url = f"{viewset.get_service().endpoint}{record.pk}/" + return url diff --git a/tests/etl/test_etl_loaders.py b/tests/etl/test_etl_loaders.py index 4a976f927..5fe9e9497 100644 --- a/tests/etl/test_etl_loaders.py +++ b/tests/etl/test_etl_loaders.py @@ -26,6 +26,8 @@ def pytest_generate_tests(metafunc): # m.append(pytest.param(model.loader, marks=pytest.mark.xfail)) if model._meta.app_label == 'prp': m.append(pytest.param(model.loader, marks=pytest.mark.skip)) + elif model._meta.app_label == 'unpp': + m.append(pytest.param(model.loader, marks=pytest.mark.skip)) elif model._meta.app_label == 'rapidpro': m.append(pytest.param(model.loader, marks=pytest.mark.skip)) else: From 9dfa22c468011fb893703085b00629aad8e5342e Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 4 Jun 2020 12:14:52 -0400 Subject: [PATCH 07/12] Lint cleanup --- src/etools_datamart/apps/sources/unpp/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/etools_datamart/apps/sources/unpp/admin.py b/src/etools_datamart/apps/sources/unpp/admin.py index a70c30cc6..9da6a05b5 100644 --- a/src/etools_datamart/apps/sources/unpp/admin.py +++ b/src/etools_datamart/apps/sources/unpp/admin.py @@ -323,4 +323,3 @@ class SanctionslistSanctionednamematchAdmin(DatamartSourceModelAdmin, ModelAdmin @register(models.SequencesSequence) class SequencesSequenceAdmin(DatamartSourceModelAdmin, ModelAdmin): list_filter = [] - From db5f4ec46a7c68b51e9eced3b3256f169c2c45c9 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 5 Jun 2020 09:56:48 -0400 Subject: [PATCH 08/12] Add unpp application endpoint --- src/etools_datamart/api/endpoints/__init__.py | 1 + .../api/endpoints/unpp/__init__.py | 1 + .../api/endpoints/unpp/application.py | 25 +++++++++++++++++++ src/etools_datamart/api/urls.py | 4 +++ 4 files changed, 31 insertions(+) create mode 100644 src/etools_datamart/api/endpoints/unpp/__init__.py create mode 100644 src/etools_datamart/api/endpoints/unpp/application.py diff --git a/src/etools_datamart/api/endpoints/__init__.py b/src/etools_datamart/api/endpoints/__init__.py index ff45b1ea8..3208448de 100644 --- a/src/etools_datamart/api/endpoints/__init__.py +++ b/src/etools_datamart/api/endpoints/__init__.py @@ -4,3 +4,4 @@ from .prp import * # noqa from .system import * # noqa from .unicef import * # noqa +from .unpp import * # noqa diff --git a/src/etools_datamart/api/endpoints/unpp/__init__.py b/src/etools_datamart/api/endpoints/unpp/__init__.py new file mode 100644 index 000000000..23149b063 --- /dev/null +++ b/src/etools_datamart/api/endpoints/unpp/__init__.py @@ -0,0 +1 @@ +from .application import ApplicationViewSet diff --git a/src/etools_datamart/api/endpoints/unpp/application.py b/src/etools_datamart/api/endpoints/unpp/application.py new file mode 100644 index 000000000..cb132dcd9 --- /dev/null +++ b/src/etools_datamart/api/endpoints/unpp/application.py @@ -0,0 +1,25 @@ +from unicef_rest_framework.ds import DynamicSerializerFilter +from unicef_rest_framework.ordering import OrderingFilter + +from etools_datamart.api.endpoints.common import DataMartViewSet +from etools_datamart.api.endpoints.datamart.serializers import DataMartSerializer +from etools_datamart.api.filtering import DatamartQueryStringFilterBackend +from etools_datamart.apps.mart.unpp import models + + +class ApplicationSerializer(DataMartSerializer): + class Meta(DataMartSerializer.Meta): + model = models.Application + exclude = None + fields = '__all__' + + +class ApplicationViewSet(DataMartViewSet): + serializer_class = ApplicationSerializer + queryset = models.Application.objects.all() + serializers_fieldsets = {'std': ApplicationSerializer, } + filter_backends = [ + DatamartQueryStringFilterBackend, + OrderingFilter, + DynamicSerializerFilter, + ] diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index 31cb3fffe..3268c5d7e 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -67,9 +67,13 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'system/monitor', endpoints.MonitorViewSet) +router.register(r'unpp/application', endpoints.ApplicationViewSet) + from etools_datamart.apps.sources.source_prp import api_urls # noqa isort:skip from etools_datamart.apps.sources.source_prp.backward_api_urls import backward_compatible_router # noqa isort:skip +from etools_datamart.apps.sources.unpp import api_urls # noqa isort:skip + from .endpoints.rapidpro import _urls_ # noqa isort:skip urlpatterns = [ From dc287445042e1024b76436845a012bb1a75097b9 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 13 Aug 2020 13:42:31 -0400 Subject: [PATCH 09/12] Update name of UNPP serializer to prevent clash --- src/etools_datamart/apps/sources/unpp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/sources/unpp/api.py b/src/etools_datamart/apps/sources/unpp/api.py index 3cb937e7c..021a454eb 100644 --- a/src/etools_datamart/apps/sources/unpp/api.py +++ b/src/etools_datamart/apps/sources/unpp/api.py @@ -158,14 +158,14 @@ class NotificationNotifieduserViewSet(URFReadOnlyModelViewSet): queryset = models.NotificationNotifieduser.objects.all() -class PartnerPartnerSerializer(serializers.ModelSerializer): +class UNPPPartnerPartnerSerializer(serializers.ModelSerializer): class Meta: model = models.PartnerPartner exclude = () class PartnerPartnerViewSet(URFReadOnlyModelViewSet): - serializer_class = PartnerPartnerSerializer + serializer_class = UNPPPartnerPartnerSerializer queryset = models.PartnerPartner.objects.all() From 98caccd549ae831e055af13d9e086577476296c8 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 13 Aug 2020 14:56:39 -0400 Subject: [PATCH 10/12] Clean up url imports --- src/etools_datamart/api/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index 3268c5d7e..2935aaf1a 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -69,10 +69,10 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'unpp/application', endpoints.ApplicationViewSet) -from etools_datamart.apps.sources.source_prp import api_urls # noqa isort:skip -from etools_datamart.apps.sources.source_prp.backward_api_urls import backward_compatible_router # noqa isort:skip +import etools_datamart.apps.sources.source_prp.api_urls # noqa isort:skip +import etools_datamart.apps.sources.source_prp.backward_api_urls.backward_compatible_router # noqa isort:skip -from etools_datamart.apps.sources.unpp import api_urls # noqa isort:skip +import etools_datamart.apps.sources.unpp.api_urls # noqa isort:skip from .endpoints.rapidpro import _urls_ # noqa isort:skip From e70949493e5c17c5aa67b9219ee84a40e7dd3b9d Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 13 Aug 2020 15:06:52 -0400 Subject: [PATCH 11/12] Correct urls import --- src/etools_datamart/api/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index 2935aaf1a..548882407 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -70,7 +70,7 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'unpp/application', endpoints.ApplicationViewSet) import etools_datamart.apps.sources.source_prp.api_urls # noqa isort:skip -import etools_datamart.apps.sources.source_prp.backward_api_urls.backward_compatible_router # noqa isort:skip +from etools_datamart.apps.sources.source_prp.backward_api_urls import backward_compatible_router # noqa isort:skip import etools_datamart.apps.sources.unpp.api_urls # noqa isort:skip From 6c9cc10448b792e0b731a7cc6e8f552715e5ec01 Mon Sep 17 00:00:00 2001 From: Domenico Date: Fri, 12 Feb 2021 18:38:15 +0100 Subject: [PATCH 12/12] clean --- setup.cfg | 2 +- src/etools_datamart/apps/mart/prp/models.py | 9 +++------ .../apps/mart/unpp/migrations/0001_initial.py | 2 +- tests/_test_lib/test_utilities/factories/__init__.py | 2 +- tests/api/interfaces/unpp.py | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5c2c5c5c2..886a62535 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,4 +25,4 @@ max-complexity = 19 max-line-length = 160 filename = .py exclude = .tox,migrations,.git,docs,diff_match_patch.py,deploy/**,settings -ignore = E731 +ignore = E731, W504 diff --git a/src/etools_datamart/apps/mart/prp/models.py b/src/etools_datamart/apps/mart/prp/models.py index 2aec48fa0..7e8ac380d 100644 --- a/src/etools_datamart/apps/mart/prp/models.py +++ b/src/etools_datamart/apps/mart/prp/models.py @@ -47,14 +47,12 @@ """ from ast import literal_eval -from django.db import models, transaction +from django.db import models from django.db.models import JSONField, Q -from django.utils import timezone from strategy_field.utils import get_attr -from etools_datamart.apps.etl.exceptions import MaxRecordsException, RequiredIsMissing, RequiredIsRunning -from etools_datamart.apps.etl.loader import BaseLoader, EtlResult, logger, RUN_UNKNOWN +from etools_datamart.apps.etl.loader import CommonLoader from etools_datamart.apps.sources.source_prp.models import ( CoreCountry, CoreGatewaytype, @@ -69,7 +67,6 @@ UnicefProgrammedocumentSections, UnicefProgressreport, ) -from etools_datamart.sentry import process_exception from .base import PrpDataMartModel @@ -177,7 +174,7 @@ def get_locations(self, record: IndicatorIndicatorlocationdata, values, **kwargs levelname=location.gateway.name )) values['locations_data'] = locs - return ", ".join([l['name'] for l in locs]) + return ", ".join([loc['name'] for loc in locs]) def get_submitted_by(self, record: IndicatorIndicatorlocationdata, values, **kwargs): user = get_attr(record, 'indicator_report.progress_report.submitted_by') diff --git a/src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py b/src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py index 2401e38b3..91b5785b4 100644 --- a/src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py +++ b/src/etools_datamart/apps/mart/unpp/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.11 on 2020-06-03 17:43 import django.contrib.postgres.fields.jsonb -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/tests/_test_lib/test_utilities/factories/__init__.py b/tests/_test_lib/test_utilities/factories/__init__.py index b127fba27..a3e5c555b 100644 --- a/tests/_test_lib/test_utilities/factories/__init__.py +++ b/tests/_test_lib/test_utilities/factories/__init__.py @@ -2,5 +2,5 @@ from .data import * # noqa from .etools import * # noqa from .prp import * # noqa -from .unpp import * # noqa from .rapidpro import * # noqa +from .unpp import * # noqa diff --git a/tests/api/interfaces/unpp.py b/tests/api/interfaces/unpp.py index 044cbab10..25012d5d3 100644 --- a/tests/api/interfaces/unpp.py +++ b/tests/api/interfaces/unpp.py @@ -22,8 +22,8 @@ def client(self): def frozenfixture2(use_request=False): def deco(func): - from drf_api_checker.utils import load_fixtures, dump_fixtures from drf_api_checker.fs import mktree + from drf_api_checker.utils import dump_fixtures, load_fixtures @wraps(func) def _inner(*args, **kwargs):