Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ch38705] Documenting SQL queries for Datamart Loaders #382

Open
wants to merge 14 commits into
base: develop
Choose a base branch
from
Open
8 changes: 8 additions & 0 deletions CHANGES
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
4.7.13.1
----
- DB Query for following loaders have been extracted;
+ location
+ office
+ partner
+ internvention

4.7.12
----
Formula for SpotCheckFindings.pending_unsupported_amount has been updated as below;
Expand Down
2 changes: 1 addition & 1 deletion src/etools_datamart/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
NAME = "etools-datamart"
VERSION = __version__ = "4.7.12"
VERSION = __version__ = "4.7.13.1"
__author__ = ""
218 changes: 218 additions & 0 deletions src/etools_datamart/apps/mart/data/models/actionpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,224 @@
RELATED_MODULE_NAMES = [c.__name__ for c in RELATED_MODULES]
RELATED_MODULE_CHOICES = zip(RELATED_MODULE_NAMES, RELATED_MODULE_NAMES)

"""
-- Set country schema
SET search_path = public, ##COUNTRY##;

-- Count for paging;
SELECT COUNT(*) AS "__count" FROM "action_points_actionpoint"

-- ActionPoint
SELECT '##COUNTRY##' AS __schema,
"action_points_actionpoint"."id",
"action_points_actionpoint"."created",
"action_points_actionpoint"."modified",
"action_points_actionpoint"."status",
"action_points_actionpoint"."description",
"action_points_actionpoint"."due_date",
"action_points_actionpoint"."date_of_completion",

"action_points_actionpoint"."assigned_by_id",
"assigned_by_auth_user"."password",
"assigned_by_auth_user"."last_login",
"assigned_by_auth_user"."is_superuser",
"assigned_by_auth_user"."username",
"assigned_by_auth_user"."first_name",
"assigned_by_auth_user"."last_name",
"assigned_by_auth_user"."email",
"assigned_by_auth_user"."is_staff",
"assigned_by_auth_user"."is_active",
"assigned_by_auth_user"."date_joined",
"assigned_by_auth_user"."middle_name",
"assigned_by_auth_user"."created",
"assigned_by_auth_user"."modified",
"assigned_by_auth_user"."preferences"

"action_points_actionpoint"."assigned_to_id",
"assigned_to_auth_user"."password",
"assigned_to_auth_user"."last_login",
"assigned_to_auth_user"."is_superuser",
"assigned_to_auth_user"."username",
"assigned_to_auth_user"."first_name",
"assigned_to_auth_user"."last_name",
"assigned_to_auth_user"."email",
"assigned_to_auth_user"."is_staff",
"assigned_to_auth_user"."is_active",
"assigned_to_auth_user"."date_joined",
"assigned_to_auth_user"."middle_name",
"assigned_to_auth_user"."created",
"assigned_to_auth_user"."modified",
"assigned_to_auth_user"."preferences"

"action_points_actionpoint"."author_id",
"author_auth_user"."password",
"author_auth_user"."last_login",
"author_auth_user"."is_superuser",
"author_auth_user"."username",
"author_auth_user"."first_name",
"author_auth_user"."last_name",
"author_auth_user"."email",
"author_auth_user"."is_staff",
"author_auth_user"."is_active",
"author_auth_user"."date_joined",
"author_auth_user"."middle_name",
"author_auth_user"."created",
"author_auth_user"."modified",
"author_auth_user"."preferences"

"action_points_actionpoint"."cp_output_id",
"reports_result"."name",
"reports_result"."code",
"reports_result"."result_type_id",
"reports_result"."sector_id",
"reports_result"."gic_code",
"reports_result"."gic_name",
"reports_result"."humanitarian_tag",
"reports_result"."level",
"reports_result"."lft",
"reports_result"."parent_id",
"reports_result"."rght",
"reports_result"."sic_code",
"reports_result"."sic_name",
"reports_result"."tree_id",
"reports_result"."vision_id",
"reports_result"."wbs",
"reports_result"."activity_focus_code",
"reports_result"."activity_focus_name",
"reports_result"."hidden",
"reports_result"."from_date",
"reports_result"."to_date",
"reports_result"."ram",
"reports_result"."country_programme_id",
"reports_result"."created",
"reports_result"."modified",
"reports_result"."humanitarian_marker_code",
"reports_result"."humanitarian_marker_name",
"reports_result"."programme_area_code",
"reports_result"."programme_area_name"

"action_points_actionpoint"."engagement_id",
"action_points_actionpoint"."intervention_id",
"action_points_actionpoint"."location_id",
"action_points_actionpoint"."office_id",

"action_points_actionpoint"."partner_id",
"partners_partnerorganization"."id",
"partners_partnerorganization"."description",
"partners_partnerorganization"."address",
"partners_partnerorganization"."email",
"partners_partnerorganization"."phone_number",
"partners_partnerorganization"."alternate_id",
"partners_partnerorganization"."alternate_name",
"partners_partnerorganization"."rating",
"partners_partnerorganization"."core_values_assessment_date",
"partners_partnerorganization"."vision_synced",
"partners_partnerorganization"."type_of_assessment",
"partners_partnerorganization"."last_assessment_date",
"partners_partnerorganization"."hidden",
"partners_partnerorganization"."deleted_flag",
"partners_partnerorganization"."total_ct_cp",
"partners_partnerorganization"."total_ct_cy",
"partners_partnerorganization"."blocked",
"partners_partnerorganization"."city",
"partners_partnerorganization"."country",
"partners_partnerorganization"."postal_code",
"partners_partnerorganization"."shared_with",
"partners_partnerorganization"."street_address",
"partners_partnerorganization"."hact_values",
"partners_partnerorganization"."created",
"partners_partnerorganization"."modified",
"partners_partnerorganization"."net_ct_cy",
"partners_partnerorganization"."reported_cy",
"partners_partnerorganization"."total_ct_ytd",
"partners_partnerorganization"."basis_for_risk_rating",
"partners_partnerorganization"."manually_blocked",
"partners_partnerorganization"."outstanding_dct_amount_6_to_9_months_usd",
"partners_partnerorganization"."outstanding_dct_amount_more_than_9_months_usd",
"partners_partnerorganization"."highest_risk_rating_name",
"partners_partnerorganization"."highest_risk_rating_type",
"partners_partnerorganization"."psea_assessment_date",
"partners_partnerorganization"."sea_risk_rating_name",
"partners_partnerorganization"."lead_office_id",
"partners_partnerorganization"."lead_section_id",

"partners_partnerorganization"."organization_id"
"organizations_organization"."created",
"organizations_organization"."modified",
"organizations_organization"."name",
"organizations_organization"."vendor_number",
"organizations_organization"."organization_type",
"organizations_organization"."cso_type",
"organizations_organization"."short_name",
"organizations_organization"."other",
"organizations_organization"."parent_id"


"action_points_actionpoint"."section_id",

"action_points_actionpoint"."tpm_activity_id",

"action_points_actionpoint"."high_priority",

"action_points_actionpoint"."travel_activity_id",
"t2f_travelactivity"."travel_type",
"t2f_travelactivity"."date",
"t2f_travelactivity"."partner_id",
"t2f_travelactivity"."partnership_id",
"t2f_travelactivity"."primary_traveler_id",
"t2f_travelactivity"."result_id",


"action_points_actionpoint"."category_id",

"action_points_actionpoint"."psea_assessment_id",

"action_points_actionpoint"."reference_number",

"action_points_actionpoint"."monitoring_activity_id"

FROM "action_points_actionpoint"
"assigned_by_auth_user"
"assigned_to_auth_user"
"assigned_to_auth_user"
"reports_result"
"partners_partnerorganization"
"organizations_organization"
"t2f_travelactivity"

ORDER BY "action_points_actionpoint"."id" ASC
LIMIT ##PAGE_SIZE## OFFSET ##PAGE_OFFSET##;

--

--
SELECT "django_content_type"."id",
"django_content_type"."app_label",
"django_content_type"."model"
FROM "django_content_type"
WHERE ("django_content_type"."app_label" = 'action_points'
AND "django_content_type"."model" = 'actionpoint')

SELECT '##COUNTRY##' AS __schema,
"django_comments"."id",
"django_comments"."object_pk",
"django_comments"."user_name",
"django_comments"."user_email",
"django_comments"."user_url",
"django_comments"."comment",
"django_comments"."submit_date",
"django_comments"."ip_address",
"django_comments"."is_public",
"django_comments"."is_removed",
"django_comments"."content_type_id",
"django_comments"."site_id",
"django_comments"."user_id"
FROM "django_comments"
WHERE ("django_comments"."content_type_id" = 270
AND "django_comments"."object_pk" = '2')

"""


class ActionPointLoader(EtoolsLoader):
def get_reference_number(self, record: ActionPointsActionpoint, values: dict, **kwargs):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,80 @@


class AttachmentLoader(EtoolsLoader):
"""
Load Queries:

-- Set country schema
SET search_path = public, ##COUNTRY##;

-- Count for paging;
SELECT COUNT(*) AS "__count" FROM "attachments_attachmentflat"

SELECT '##COUNTRY##' AS __schema,
"attachments_attachmentflat"."id",
"attachments_attachmentflat"."partner",
"attachments_attachmentflat"."partner_type",
"attachments_attachmentflat"."vendor_number",
"attachments_attachmentflat"."pd_ssfa_number",
"attachments_attachmentflat"."file_type",
"attachments_attachmentflat"."file_link",
"attachments_attachmentflat"."uploaded_by",
"attachments_attachmentflat"."attachment_id",
"attachments_attachmentflat"."filename",
"attachments_attachmentflat"."agreement_reference_number",
"attachments_attachmentflat"."object_link",
"attachments_attachmentflat"."source",
"attachments_attachmentflat"."pd_ssfa",
"attachments_attachmentflat"."created",
"attachments_attachmentflat"."ip_address",

"unicef_attachments_attachment"."id",
"unicef_attachments_attachment"."created",
"unicef_attachments_attachment"."modified",
"unicef_attachments_attachment"."file",
"unicef_attachments_attachment"."hyperlink",
"unicef_attachments_attachment"."object_id",
"unicef_attachments_attachment"."code",
"unicef_attachments_attachment"."content_type_id",
"unicef_attachments_attachment"."file_type_id",
"unicef_attachments_attachment"."uploaded_by_id",
"unicef_attachments_attachment"."ip_address",

"django_content_type"."id",
"django_content_type"."app_label",
"django_content_type"."model",

"unicef_attachments_filetype"."id",
"unicef_attachments_filetype"."order",
"unicef_attachments_filetype"."name",
"unicef_attachments_filetype"."label",
"unicef_attachments_filetype"."code",
"unicef_attachments_filetype"."group",

"auth_user"."id", "auth_user"."password",
"auth_user"."last_login",
"auth_user"."is_superuser",
"auth_user"."username",
"auth_user"."first_name",
"auth_user"."last_name",
"auth_user"."email",
"auth_user"."is_staff",
"auth_user"."is_active",
"auth_user"."date_joined",
"auth_user"."middle_name",
"auth_user"."created",
"auth_user"."modified",
"auth_user"."preferences"

FROM "attachments_attachmentflat"
INNER JOIN "unicef_attachments_attachment" ON ("attachments_attachmentflat"."attachment_id" = "unicef_attachments_attachment"."id")
LEFT OUTER JOIN "django_content_type" ON ("unicef_attachments_attachment"."content_type_id" = "django_content_type"."id")
LEFT OUTER JOIN "unicef_attachments_filetype" ON ("unicef_attachments_attachment"."file_type_id" = "unicef_attachments_filetype"."id")
LEFT OUTER JOIN "auth_user" ON ("unicef_attachments_attachment"."uploaded_by_id" = "auth_user"."id")
ORDER BY "attachments_attachmentflat"."id" ASC
LIMIT ##PAGE_SIZE## OFFSET ##PAGE_OFFSET##
"""

def get_queryset(self):
return self.config.source.objects.select_related(
"attachment",
Expand Down
Loading