Merge branch 'release/3.54'

CHANGES

@@ -1,3 +1,8 @@
+3.54
+
+added audit/financial-findings/ endpoint
+added interventions-planned-visits endpoint
+
 3.53.1
 ------
 * fixed MR PV

src/etools_datamart/__init__.py

@@ -1,3 +1,3 @@
 NAME = 'etools-datamart'
-VERSION = __version__ = '3.53.1'
+VERSION = __version__ = '3.54'
 __author__ = ''

src/etools_datamart/api/endpoints/datamart/__init__.py

@@ -3,6 +3,7 @@
 from .attachment import AttachmentViewSet
 from .audit_audit import AuditViewSet
 from .audit_engagement import EngagementDetailViewSet, EngagementViewSet
+from .audit_financial_finding import AuditFinancialFindingViewSet
 from .audit_micro_assessment import MicroAssessmentViewSet
 from .audit_result import AuditResultViewSet
 from .audit_special import AuditSpecialViewSet
@@ -18,6 +19,7 @@
 from .intervention_country_programme import InterventionCPViewSet
 from .intervention_epd import InterventionEPDViewSet
 from .intervention_management_budget import InterventionBudgetMgmtViewSet
+from .intervention_planned_visits import InterventionPlannedVisitsViewSet
 from .intervention_review import InterventionReviewViewSet
 from .location import LocationSiteViewSet, LocationViewSet
 from .office import OfficeViewSet

src/etools_datamart/api/endpoints/datamart/audit_financial_finding.py

@@ -0,0 +1,34 @@
+from django import forms
+
+from unicef_rest_framework.forms import CleareableSelect2MultipleChoiceField
+
+from etools_datamart.api.endpoints.datamart.serializers import DataMartSerializer
+from etools_datamart.apps.mart.data import models
+from etools_datamart.apps.sources.etools.enrichment.consts import AuditEngagementConsts
+
+from .. import common
+
+
+class AuditFinancialFindingSerializer(DataMartSerializer):
+    class Meta(DataMartSerializer.Meta):
+        model = models.AuditFinancialFinding
+
+
+class AuditFinancialFindingFilterForm(forms.Form):
+    status__in = CleareableSelect2MultipleChoiceField(required=False,
+                                                      choices=AuditEngagementConsts.DISPLAY_STATUSES,
+                                                      )
+
+    def __init__(self, data=None, files=None, auto_id='id_%s', prefix=None, initial=None, *args, **kwargs):
+        filters = data.copy()
+        if 'status__in' in filters:
+            filters.setlist('status__in', data['status__in'].split(','))
+        super().__init__(filters, files, auto_id, prefix, initial, *args, **kwargs)
+
+
+class AuditFinancialFindingViewSet(common.DataMartViewSet):
+    serializer_class = AuditFinancialFindingSerializer
+    queryset = models.AuditFinancialFinding.objects.all()
+    filter_fields = ('status',)
+    serializers_fieldsets = {'std': AuditFinancialFindingSerializer}
+    querystringfilter_form_base_class = AuditFinancialFindingFilterForm

src/etools_datamart/api/endpoints/datamart/intervention_planned_visits.py

@@ -0,0 +1,15 @@
+from etools_datamart.apps.mart.data import models
+
+from .. import common
+from .serializers import DataMartSerializer
+
+
+class InterventionPlannedVisitsSerializer(DataMartSerializer):
+    class Meta(DataMartSerializer.Meta):
+        model = models.InterventionPlannedVisits
+        exclude = ('seen', 'source_id',)
+
+
+class InterventionPlannedVisitsViewSet(common.DataMartViewSet):
+    serializer_class = InterventionPlannedVisitsSerializer
+    queryset = models.InterventionPlannedVisits.objects.all()

src/etools_datamart/api/urls.py

@@ -27,6 +27,7 @@ class ReadOnlyRouter(APIReadOnlyRouter):
 router.register(r'datamart/audit/audit', endpoints.AuditViewSet)
 router.register(r'datamart/audit/engagements', endpoints.EngagementViewSet)
 router.register(r'datamart/audit/engagement-details', endpoints.EngagementDetailViewSet, basename="engagement-details")
+router.register(r'datamart/audit/financial-findings', endpoints.AuditFinancialFindingViewSet)
 router.register(r'datamart/audit/micro-assessment', endpoints.MicroAssessmentViewSet)
 router.register(r'datamart/audit/results', endpoints.AuditResultViewSet)
 router.register(r'datamart/audit/special-audit', endpoints.AuditSpecialViewSet)
@@ -49,6 +50,7 @@ class ReadOnlyRouter(APIReadOnlyRouter):
 router.register(r'datamart/interventions-country-programmes', endpoints.InterventionCPViewSet)
 router.register(r'datamart/interventions-epd', endpoints.InterventionEPDViewSet)
 router.register(r'datamart/interventions-management-budget', endpoints.InterventionBudgetMgmtViewSet)
+router.register(r'datamart/interventions-planned-visits', endpoints.InterventionPlannedVisitsViewSet)
 router.register(r'datamart/interventions-review', endpoints.InterventionReviewViewSet)
 
 router.register(r'datamart/pd-indicators', endpoints.PDIndicatorViewSet)

src/etools_datamart/apps/mart/data/admin.py

@@ -202,6 +202,13 @@ class InterventionManagementBudgetAdmin(DataModelAdmin, TruncateTableMixin):
     search_fields = ('pd_number', 'partner')
 
 
+@register(models.InterventionPlannedVisits)
+class InterventionPlannedVisitsAdmin(DataModelAdmin, TruncateTableMixin):
+    list_display = ('partner_vendor_number', 'partner_name', 'pd_status', 'pd_reference_number')
+    list_filter = (SchemaFilter, )
+    search_fields = ('partner_vendor_number', 'partner_name', 'pd_reference_number')
+
+
 @register(models.InterventionReview)
 class InterventionReviewAdmin(DataModelAdmin, TruncateTableMixin):
     list_display = ('country_name', 'pd_number', 'partner')
@@ -480,6 +487,13 @@ class AuditAdmin(DataModelAdmin):
     search_fields = ('reference_number', 'source_id')
 
 
+@register(models.AuditFinancialFinding)
+class AuditFinancialFindingAdmin(DataModelAdmin):
+    list_display = ('audit_reference_number', 'audit_status', 'partner_name')
+    list_filter = ('partner_name', 'audit_status', )
+    search_fields = ('partner_name', 'audit_reference_number', 'source_id')
+
+
 @register(models.AuditSpecial)
 class AuditSpecialAdmin(DataModelAdmin):
     list_display = ('reference_number', 'engagement_type', 'status', 'sections')

src/etools_datamart/apps/mart/data/migrations/0018_interventionplannedvisits.py

@@ -0,0 +1,40 @@
+# Generated by Django 4.1.5 on 2023-03-20 16:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0017_merge_20230206_2300'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='InterventionPlannedVisits',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source_id', models.IntegerField(blank=True, db_index=True, null=True)),
+                ('last_modify_date', models.DateTimeField(auto_now=True)),
+                ('seen', models.DateTimeField(blank=True, null=True)),
+                ('country_name', models.CharField(max_length=100)),
+                ('schema_name', models.CharField(db_index=True, max_length=63)),
+                ('area_code', models.CharField(db_index=True, max_length=10)),
+                ('created', models.DateTimeField(blank=True, null=True)),
+                ('modified', models.DateTimeField(blank=True, null=True)),
+                ('partner_vendor_number', models.CharField(blank=True, max_length=100, null=True)),
+                ('partner_name', models.CharField(max_length=200, null=True)),
+                ('pd_status', models.CharField(choices=[('draft', 'Draft'), ('signed', 'Signed'), ('active', 'Active'), ('ended', 'Ended'), ('closed', 'Closed'), ('suspended', 'Suspended'), ('terminated', 'Terminated'), ('cancelled', 'Cancelled')], db_index=True, max_length=32, null=True)),
+                ('pd_reference_number', models.CharField(max_length=100, null=True)),
+                ('year', models.IntegerField()),
+                ('programmatic_q1', models.IntegerField(default=0)),
+                ('programmatic_q2', models.IntegerField(default=0)),
+                ('programmatic_q3', models.IntegerField(default=0)),
+                ('programmatic_q4', models.IntegerField(default=0)),
+            ],
+            options={
+                'verbose_name_plural': 'Intervention Planned Visits',
+                'ordering': ('-created',),
+            },
+        ),
+    ]

src/etools_datamart/apps/mart/data/migrations/0019_auditfinancialfinding.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.1.5 on 2023-03-22 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0018_interventionplannedvisits'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AuditFinancialFinding',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source_id', models.IntegerField(blank=True, db_index=True, null=True)),
+                ('last_modify_date', models.DateTimeField(auto_now=True)),
+                ('seen', models.DateTimeField(blank=True, null=True)),
+                ('country_name', models.CharField(max_length=100)),
+                ('schema_name', models.CharField(db_index=True, max_length=63)),
+                ('area_code', models.CharField(db_index=True, max_length=10)),
+                ('created', models.DateField(auto_now=True)),
+                ('partner_vendor_number', models.CharField(blank=True, max_length=50, null=True)),
+                ('partner_name', models.CharField(blank=True, max_length=255, null=True)),
+                ('audit_reference_number', models.CharField(blank=True, max_length=100, null=True)),
+                ('audit_status', models.CharField(blank=True, choices=[('partner_contacted', 'IP Contacted'), ('field_visit', 'Field Visit'), ('draft_issued_to_partner', 'Draft Report Issued to IP'), ('comments_received_by_partner', 'Comments Received from IP'), ('draft_issued_to_unicef', 'Draft Report Issued to UNICEF'), ('comments_received_by_unicef', 'Comments Received from UNICEF'), ('report_submitted', 'Report Submitted'), ('final', 'Final Report'), ('cancelled', 'Cancelled')], db_index=True, max_length=30, null=True)),
+                ('title', models.CharField(choices=[('no-supporting-documentation', 'No supporting documentation'), ('insufficient-supporting-documentation', 'Insufficient supporting documentation'), ('cut-off-error', 'Cut-off error'), ('expenditure-not-for-project-purposes', 'Expenditure not for project purposes'), ('no-proof-of-payment', 'No proof of payment'), ('no-proof-of-goods-services-received', 'No proof of goods / services received'), ('vat-incorrectly-claimed', 'VAT incorrectly claimed'), ('dsa-rates-exceeded', 'DSA rates exceeded'), ('unreasonable-price', 'Unreasonable price'), ('bank-interest-not-reported', 'Bank interest not reported'), ('support-costs-incorrectly-calculated', 'Support costs incorrectly calculated'), ('expenditure-claimed-but-activities-not-undertaken', 'Expenditure claimed but activities not undertaken'), ('advance-claimed-as-expenditure', 'Advance claimed as expenditure'), ('commitments-treated-as-expenditure', 'Commitments treated as expenditure'), ('ineligible-salary-costs', 'Ineligible salary costs'), ('ineligible-costs-other', 'Ineligible costs (other)')], max_length=255)),
+                ('local_amount', models.DecimalField(decimal_places=2, max_digits=20)),
+                ('amount', models.DecimalField(decimal_places=2, max_digits=20)),
+                ('description', models.TextField()),
+                ('recommendation', models.TextField(blank=True)),
+                ('ip_comments', models.TextField(blank=True)),
+            ],
+            options={
+                'ordering': ('id',),
+            },
+        ),
+    ]

src/etools_datamart/apps/mart/data/models/__init__.py

@@ -2,6 +2,7 @@
 from .attachments_attachment import Attachment  # noqa
 from .audit_audit import Audit  # noqa
 from .audit_engagement import Engagement  # noqa
+from .audit_financial_finding import AuditFinancialFinding  # noqa
 from .audit_micro_assessment import MicroAssessment
 from .audit_result import AuditResult  # noqa
 from .audit_special import AuditSpecial  # noqa
@@ -18,6 +19,7 @@
 from .intervention_country_programme import InterventionCountryProgramme  # noqa
 from .intervention_epd import InterventionEPD  # noqa
 from .intervention_management_budget import InterventionManagementBudget  # noqa
+from .intervention_planned_visits import InterventionPlannedVisits
 from .intervention_review import InterventionReview  # noqa
 from .location import GeoName, Location  # noqa
 from .partner import Partner  # noqa

src/etools_datamart/apps/mart/data/models/audit_financial_finding.py

@@ -0,0 +1,60 @@
+from django.db import models
+
+from etools_datamart.apps.mart.data.loader import EtoolsLoader
+from etools_datamart.apps.mart.data.models import Audit
+from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel
+from etools_datamart.apps.sources.etools.enrichment.consts import AuditEngagementConsts, AuditFinancialFindingsConsts
+from etools_datamart.apps.sources.etools.models import AuditFinancialfinding
+
+
+class AuditFinancialfindingLoader(EtoolsLoader):
+    def process_country(self):
+        country = self.context['country']
+        for record in self.filter_queryset(self.get_queryset()):
+            try:
+                audit = Audit.objects.get(source_id=record.audit_id, schema_name=country.schema_name)
+                filters = self.config.key(self, record)
+                values = self.get_values(record)
+                values['partner_vendor_number'] = audit.auditor_number
+                values['partner_name'] = audit.auditor
+                values['audit_reference_number'] = audit.reference_number
+                values['audit_status'] = audit.status
+                op = self.process_record(filters, values)
+                self.increment_counter(op)
+            except Audit.DoesNotExist:
+                pass
+
+    def get_title(self, record, values, field_name):
+        return AuditFinancialFindingsConsts.TITLE_CHOICES[getattr(record, field_name)]
+
+
+class AuditFinancialFinding(EtoolsDataMartModel):
+    created = models.DateField(auto_now=True)
+
+    partner_vendor_number = models.CharField(max_length=50, blank=True, null=True)
+    partner_name = models.CharField(max_length=255, blank=True, null=True)
+
+    audit_reference_number = models.CharField(max_length=100, blank=True, null=True)
+    audit_status = models.CharField(max_length=30, blank=True, null=True,
+                                    choices=AuditEngagementConsts.DISPLAY_STATUSES,
+                                    db_index=True)
+
+    title = models.CharField(max_length=255, choices=AuditFinancialFindingsConsts.TITLE_CHOICES)
+    local_amount = models.DecimalField(max_digits=20, decimal_places=2)
+    amount = models.DecimalField(max_digits=20, decimal_places=2)
+    description = models.TextField()
+    recommendation = models.TextField(blank=True)
+    ip_comments = models.TextField(blank=True)
+
+    loader = AuditFinancialfindingLoader()
+
+    class Meta:
+        ordering = ("id",)
+
+    class Options:
+        source = AuditFinancialfinding
+        depends = (Audit, )
+        key = lambda loader, record: dict(
+            schema_name=loader.context['country'].schema_name,
+            source_id=record.id,
+        )

src/etools_datamart/apps/mart/data/models/intervention_planned_visits.py

@@ -0,0 +1,50 @@
+from etools_datamart.apps.mart.data.loader import EtoolsLoader
+from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel
+from etools_datamart.apps.mart.data.models.intervention import Intervention
+from etools_datamart.apps.sources.etools.enrichment.consts import PartnersInterventionConst
+from etools_datamart.apps.sources.etools.models import models, PartnersInterventionplannedvisits
+
+
+class InterventionPlannedVisits(EtoolsDataMartModel):
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
+
+    partner_vendor_number = models.CharField(max_length=100, blank=True, null=True)
+    partner_name = models.CharField(max_length=200, null=True)
+    pd_status = models.CharField(max_length=32, null=True, db_index=True,
+                                 choices=PartnersInterventionConst.STATUSES)
+    pd_reference_number = models.CharField(max_length=100, null=True)
+
+    year = models.IntegerField()
+    programmatic_q1 = models.IntegerField(default=0)
+    programmatic_q2 = models.IntegerField(default=0)
+    programmatic_q3 = models.IntegerField(default=0)
+    programmatic_q4 = models.IntegerField(default=0)
+
+    loader = EtoolsLoader()
+
+    class Meta:
+        ordering = ("-created",)
+        verbose_name_plural = "Intervention Planned Visits"
+
+    class Options:
+        source = PartnersInterventionplannedvisits
+        depends = (Intervention,)
+        queryset = lambda: PartnersInterventionplannedvisits.objects.select_related(
+            'intervention',
+            'intervention__agreement',
+            'intervention__agreement__partner',
+        )
+        key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name,
+                                          source_id=record.pk)
+        mapping = dict(
+            partner_vendor_number='intervention.agreement.partner.vendor_number',
+            partner_name='intervention.agreement.partner.name',
+            pd_status='intervention.status',
+            pd_reference_number='intervention.reference_number',
+            year='=',
+            programmatic_q1='=',
+            programmatic_q2='=',
+            programmatic_q3='=',
+            programmatic_q4='='
+        )

src/etools_datamart/apps/sources/etools/enrichment/consts.py

@@ -229,6 +229,27 @@ class AuditEngagementConsts:
     }
 
 
+class AuditFinancialFindingsConsts:
+    TITLE_CHOICES = Choices(
+        ('no-supporting-documentation', _('No supporting documentation')),
+        ('insufficient-supporting-documentation', _('Insufficient supporting documentation')),
+        ('cut-off-error', _('Cut-off error')),
+        ('expenditure-not-for-project-purposes', _('Expenditure not for project purposes')),
+        ('no-proof-of-payment', _('No proof of payment')),
+        ('no-proof-of-goods-services-received', _('No proof of goods / services received')),
+        ('vat-incorrectly-claimed', _('VAT incorrectly claimed')),
+        ('dsa-rates-exceeded', _('DSA rates exceeded')),
+        ('unreasonable-price', _('Unreasonable price')),
+        ('bank-interest-not-reported', _('Bank interest not reported')),
+        ('support-costs-incorrectly-calculated', _('Support costs incorrectly calculated')),
+        ('expenditure-claimed-but-activities-not-undertaken', _('Expenditure claimed but activities not undertaken')),
+        ('advance-claimed-as-expenditure', _('Advance claimed as expenditure')),
+        ('commitments-treated-as-expenditure', _('Commitments treated as expenditure')),
+        ('ineligible-salary-costs', _('Ineligible salary costs')),
+        ('ineligible-costs-other', _('Ineligible costs (other)')),
+    )
+
+
 class ActionPointConsts:
     MODULE_CHOICES = CategoryConsts.MODULE_CHOICES
 

src/etools_datamart/apps/sources/etools/models/tenant.py

@@ -1326,13 +1326,13 @@ class Meta:
 class PartnersInterventionplannedvisits(models.TenantModel):
     id = models.IntegerField(primary_key=True)
     year = models.IntegerField()
-    programmatic_q4 = models.IntegerField()
     intervention = models.ForeignKey(PartnersIntervention, models.DO_NOTHING, related_name='PartnersInterventionplannedvisits_intervention')
     created = models.DateTimeField()
     modified = models.DateTimeField()
-    programmatic_q1 = models.IntegerField()
-    programmatic_q2 = models.IntegerField()
-    programmatic_q3 = models.IntegerField()
+    programmatic_q1 = models.IntegerField(default=0)
+    programmatic_q2 = models.IntegerField(default=0)
+    programmatic_q3 = models.IntegerField(default=0)
+    programmatic_q4 = models.IntegerField(default=0)
 
     class Meta:
         managed = False