Binary_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'local[*]' - instructing to use all the cores (*) on a local machine\n",
    "# 'appName' - will be given a random name unless specified, like Docker container image name\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "\n",
    "spark = SparkSession.builder. \\\n",
    "        master('local[*]').\\\n",
    "        appName('credit risk').\\\n",
    "        getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data source CSV file must reside in the same folder\n",
    "# The 'german.data-numeric' file delimits columns with 2 spaces ('  ')\n",
    "#   Without data scrubbing, \"IllegalArgumentException: 'Delimiter cannot be more than one character:   '\"\n",
    "df = spark.read.csv('german.data',sep=' ',inferSchema=False,header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 21)\n",
      "root\n",
      " |-- _c0: string (nullable = true)\n",
      " |-- _c1: string (nullable = true)\n",
      " |-- _c2: string (nullable = true)\n",
      " |-- _c3: string (nullable = true)\n",
      " |-- _c4: string (nullable = true)\n",
      " |-- _c5: string (nullable = true)\n",
      " |-- _c6: string (nullable = true)\n",
      " |-- _c7: string (nullable = true)\n",
      " |-- _c8: string (nullable = true)\n",
      " |-- _c9: string (nullable = true)\n",
      " |-- _c10: string (nullable = true)\n",
      " |-- _c11: string (nullable = true)\n",
      " |-- _c12: string (nullable = true)\n",
      " |-- _c13: string (nullable = true)\n",
      " |-- _c14: string (nullable = true)\n",
      " |-- _c15: string (nullable = true)\n",
      " |-- _c16: string (nullable = true)\n",
      " |-- _c17: string (nullable = true)\n",
      " |-- _c18: string (nullable = true)\n",
      " |-- _c19: string (nullable = true)\n",
      " |-- _c20: string (nullable = true)\n",
      "\n",
      "+---+---+---+---+----+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+\n",
      "|_c0|_c1|_c2|_c3| _c4|_c5|_c6|_c7|_c8| _c9|_c10|_c11|_c12|_c13|_c14|_c15|_c16|_c17|_c18|_c19|_c20|\n",
      "+---+---+---+---+----+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+\n",
      "|A11|  6|A34|A43|1169|A65|A75|  4|A93|A101|   4|A121|  67|A143|A152|   2|A173|   1|A192|A201|   1|\n",
      "|A12| 48|A32|A43|5951|A61|A73|  2|A92|A101|   2|A121|  22|A143|A152|   1|A173|   1|A191|A201|   2|\n",
      "|A14| 12|A34|A46|2096|A61|A74|  2|A93|A101|   3|A121|  49|A143|A152|   1|A172|   2|A191|A201|   1|\n",
      "+---+---+---+---+----+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "PRINT_ROW_NUM = 3\n",
    "COL_TOTAL_NUM = len(df.columns)\n",
    "\n",
    "print((df.count(),COL_TOTAL_NUM))\n",
    "df.printSchema()\n",
    "df.show(PRINT_ROW_NUM)\n",
    "\n",
    "### SAMPLE OPERATION ###\n",
    "# t = df.collect()[0]['features'] # Specify a partiuclar row-and-column intersection\n",
    "# print(t[0], t.indices, , t.indices[18], t.indices.shape[0])\n",
    "\n",
    "# col_names = []\n",
    "# row_values = []\n",
    "\n",
    "# for col_name in feature_columns:\n",
    "#     col_names.append(col_name)\n",
    "\n",
    "#     if get_dtype(df, col_name) == 'smallint':\n",
    "#         row_values.append(df.collect()[0][col_name])\n",
    "#     elif get_dtype(df, col_name) == 'vector' :\n",
    "#         row_values.append(df.collect()[0][col_name][1])    \n",
    "    \n",
    "# print(tuple(col_names))\n",
    "# print(tuple(row_values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All the columns with numbers are \"**ordinal**\" i.e., the magnitude in each column/feature means \"something\" (e.g., age, duration, credit amount and installment rate). Thus, **transformation** and **encoding** is NOT needed and they will be converted to non-string."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import ShortType # a signed 16-bit integer vs. IntegerType (a signed 32-bit integer)\n",
    "\n",
    "col_indices_of_ordinal_numbers = [2, 5, 8, 11, 13, 16, 18] # Ignore the target/label column i.e., '_c20', the last column\n",
    "col_indices_of_categorical_numbers = []\n",
    "\n",
    "def convert_integer_to_string_in_list(aList: list) -> list:\n",
    "    return [str(item) for item in aList]\n",
    "\n",
    "for col_idx in range(1, COL_TOTAL_NUM):\n",
    "    if col_idx not in col_indices_of_ordinal_numbers:\n",
    "        col_indices_of_categorical_numbers.append(col_idx)\n",
    "\n",
    "for col_name in df.schema.names:\n",
    "    # change the name of each column e.g., \"_c0\" -> \"1\"\n",
    "    new_col_num = int(col_name.replace('_c', '')) + 1\n",
    "    df = df.withColumnRenamed(col_name, str(new_col_num))\n",
    "    \n",
    "    if new_col_num in col_indices_of_ordinal_numbers:  \n",
    "        df = df.withColumn(str(new_col_num), df[str(new_col_num)].cast(ShortType()))\n",
    "        \n",
    "col_indices_of_ordinal_numbers = convert_integer_to_string_in_list(col_indices_of_ordinal_numbers)\n",
    "col_indices_of_categorical_numbers = convert_integer_to_string_in_list(col_indices_of_categorical_numbers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---+---+---+----+---+---+---+---+----+---+----+---+----+----+---+----+---+----+----+---+\n",
      "|  1|  2|  3|  4|   5|  6|  7|  8|  9|  10| 11|  12| 13|  14|  15| 16|  17| 18|  19|  20| 21|\n",
      "+---+---+---+---+----+---+---+---+---+----+---+----+---+----+----+---+----+---+----+----+---+\n",
      "|A11|  6|A34|A43|1169|A65|A75|  4|A93|A101|  4|A121| 67|A143|A152|  2|A173|  1|A192|A201|  1|\n",
      "|A12| 48|A32|A43|5951|A61|A73|  2|A92|A101|  2|A121| 22|A143|A152|  1|A173|  1|A191|A201|  2|\n",
      "|A14| 12|A34|A46|2096|A61|A74|  2|A93|A101|  3|A121| 49|A143|A152|  1|A172|  2|A191|A201|  1|\n",
      "+---+---+---+---+----+---+---+---+---+----+---+----+---+----+----+---+----+---+----+----+---+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('1', 'string'),\n",
       " ('2', 'smallint'),\n",
       " ('3', 'string'),\n",
       " ('4', 'string'),\n",
       " ('5', 'smallint'),\n",
       " ('6', 'string'),\n",
       " ('7', 'string'),\n",
       " ('8', 'smallint'),\n",
       " ('9', 'string'),\n",
       " ('10', 'string'),\n",
       " ('11', 'smallint'),\n",
       " ('12', 'string'),\n",
       " ('13', 'smallint'),\n",
       " ('14', 'string'),\n",
       " ('15', 'string'),\n",
       " ('16', 'smallint'),\n",
       " ('17', 'string'),\n",
       " ('18', 'smallint'),\n",
       " ('19', 'string'),\n",
       " ('20', 'string'),\n",
       " ('21', 'string')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.show(PRINT_ROW_NUM)\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Databricks**, the leading crowd platform for massive scale data engineering and collaborative data science with optimized Apache Spark™ clusters on AWS or Azure, just pubslihed articles [here](https://databricks-prod-cloudfront.cloud.databricks.com/public/4027ec902e239c93eaaa8714f173bcfc/5703650563260092/2738738457173388/3024994340111770/latest.html) and [there](https://databricks-prod-cloudfront.cloud.databricks.com/public/4027ec902e239c93eaaa8714f173bcfc/7274796508260238/2844662950865680/2879908466733289/latest.html) on Apache Spark MLlib Pipelines API. Let's try that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import OneHotEncoderEstimator\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "\n",
    "\n",
    "COL_SUFFIX_IDX = \"_INDEX\"\n",
    "COL_SUFFIX_VEC = \"_VEC\"\n",
    "\n",
    "stages = []\n",
    "\n",
    "for col_idx in col_indices_of_categorical_numbers:\n",
    "    # Category Indexing with StringIndexer\n",
    "    stringIndexer = StringIndexer(inputCol=col_idx, outputCol=col_idx + COL_SUFFIX_IDX)\n",
    "    \n",
    "    # Use OneHotEncoder to convert categorical variables into binary SparseVectors\n",
    "    encoder = OneHotEncoderEstimator(inputCols=[stringIndexer.getOutputCol()], outputCols=[col_idx + COL_SUFFIX_VEC])\n",
    "    \n",
    "    # Add stages. These are not run here, but will run all at once later on\n",
    "    stages += [stringIndexer, encoder]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The two columns MUST be marked as 'label' and 'features'!\n",
    "# Otherwise, \"Py4JJavaError: An error occurred while calling o1175.fit. : java.lang.IllegalArgumentException: Field \"label\" does not exist.\"\n",
    "LABEL_COL_NAME = \"label\"\n",
    "FEATURE_VECTOR_COL_NAME = \"features\"\n",
    "\n",
    "label_stringIdx = StringIndexer(inputCol=str(COL_TOTAL_NUM), outputCol=LABEL_COL_NAME)\n",
    "stages += [label_stringIdx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform all features into a vector using VectorAssembler\n",
    "assemblerInputs = [col_idx + COL_SUFFIX_IDX for col_idx in col_indices_of_categorical_numbers] + col_indices_of_ordinal_numbers\n",
    "assembler = VectorAssembler(inputCols=assemblerInputs, outputCol=FEATURE_VECTOR_COL_NAME)\n",
    "stages += [assembler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import StandardScaler\n",
    "\n",
    "\n",
    "STANDARDIZED_FEATURES_COL_NAME = 'std_features'\n",
    "\n",
    "standardScaler = StandardScaler(withMean=True, withStd=True, inputCol=FEATURE_VECTOR_COL_NAME, outputCol=STANDARDIZED_FEATURES_COL_NAME)\n",
    "stages += [standardScaler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "partialPipeline = Pipeline().setStages(stages)\n",
    "pipelineModel = partialPipeline.fit(df)\n",
    "df_pipeline = pipelineModel.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "from pyspark.mllib.evaluation import BinaryClassificationMetrics\n",
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "\n",
    "\n",
    "def print_performance_metrics(predictions):\n",
    "  # Evaluate model\n",
    "  evaluator = BinaryClassificationEvaluator(rawPredictionCol=\"rawPrediction\")\n",
    "  auc = evaluator.evaluate(predictions, {evaluator.metricName: \"areaUnderROC\"})\n",
    "  aupr = evaluator.evaluate(predictions, {evaluator.metricName: \"areaUnderPR\"})\n",
    "  print(\"auc = {}\".format(auc))\n",
    "  print(\"aupr = {}\".format(aupr))\n",
    "\n",
    "  # get rdd of predictions and labels for mllib eval metrics\n",
    "  predictionAndLabels = predictions.select(\"prediction\",\"label\").rdd\n",
    "\n",
    "  # Instantiate metrics objects\n",
    "  binary_metrics = BinaryClassificationMetrics(predictionAndLabels)\n",
    "  multi_metrics = MulticlassMetrics(predictionAndLabels)\n",
    "\n",
    "  # Area under precision-recall curve\n",
    "  print(\"Area under PR = {}\".format(binary_metrics.areaUnderPR))\n",
    "  # Area under ROC curve\n",
    "  print(\"Area under ROC = {}\".format(binary_metrics.areaUnderROC))\n",
    "  # Accuracy\n",
    "  print(\"Accuracy = {}\".format(multi_metrics.accuracy))\n",
    "  # Confusion Matrix\n",
    "  print(multi_metrics.confusionMatrix())\n",
    "  \n",
    "  # F1\n",
    "  print(\"F1 = {}\".format(multi_metrics.fMeasure()))\n",
    "  # Precision\n",
    "  print(\"Precision = {}\".format(multi_metrics.precision()))\n",
    "  # Recall\n",
    "  print(\"Recall = {}\".format(multi_metrics.recall()))\n",
    "  # FPR\n",
    "  print(\"FPR = {}\".format(multi_metrics.falsePositiveRate(0.0)))\n",
    "  # TPR\n",
    "  print(\"TPR = {}\".format(multi_metrics.truePositiveRate(0.0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1: 0.11832168009876139\n",
      "2: 0.20081984440853262\n",
      "3: 0.2759429720647131\n",
      "4: 0.3434691583369005\n",
      "5: 0.4068048276404337\n",
      "6: 0.4646192825677727\n",
      "7: 0.5212579515868058\n",
      "8: 0.5726833221543236\n",
      "9: 0.620595299353551\n",
      "10: 0.66634309561251\n",
      "11: 0.71070821099644\n",
      "12: 0.7534198111210798\n",
      "13: 0.7946167029948367\n",
      "14: 0.8334120028543393\n",
      "15: 0.8697509575845473\n",
      "16: 0.9040447458263459\n",
      "17: 0.936440526700285\n",
      "18: 0.9657619466572893\n",
      "19: 0.9871407977706779\n",
      "20: 1.0000000000000002\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from pyspark.ml.feature import PCA\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "\n",
    "\n",
    "PCA_FEATURES_COL_NAME = \"pca_features\"\n",
    "MAX_ITER_NUM = 50\n",
    "\n",
    "PCA_feature_num_list = np.arange(1, COL_TOTAL_NUM, 1)\n",
    "PCA_explainedVariances_list = []\n",
    "\n",
    "for PCA_feature_num in PCA_feature_num_list:\n",
    "    PCA_extracted = PCA(k=PCA_feature_num, inputCol=STANDARDIZED_FEATURES_COL_NAME, outputCol=PCA_FEATURES_COL_NAME)\n",
    "\n",
    "    pca_model = PCA_extracted.fit(df_pipeline)\n",
    "    df_PCA = pca_model.transform(df_pipeline)\n",
    "\n",
    "    df_train, df_test = df_PCA.randomSplit([0.7, 0.3], seed = 100)\n",
    "    \n",
    "    LR = LogisticRegression(labelCol=LABEL_COL_NAME, featuresCol=PCA_FEATURES_COL_NAME, maxIter=MAX_ITER_NUM)\n",
    "    LR_model = LR.fit(df_train)\n",
    "    LR_preds = LR_model.transform(df_test) # Scikit-Learn uses predict() method for prediction\n",
    "\n",
    "#     print(pca_model.explainedVariance) [0.11832168009876139,0.08249816430977125,0.07512312765618046]\n",
    "#     print(np.cumsum(pca_model.explainedVariance)) [0.11832168 0.20081984 0.27594297]\n",
    "\n",
    "    cumulative_explained_variance = np.cumsum(pca_model.explainedVariance)[PCA_feature_num - 1]\n",
    "\n",
    "    PCA_explainedVariances_list.append(cumulative_explained_variance)\n",
    "    print(str(PCA_feature_num) + \": \" + str(cumulative_explained_variance))\n",
    "\n",
    "#     LR_preds.select(LABEL_COL_NAME, \"prediction\", \"rawPrediction\", \"probability\").show(PRINT_ROW_NUM)\n",
    "#     print_performance_metrics(LR_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5hU5fn/8fdNXXqRXpbeQdpSFGPBhrFFo7FhAQRi7CZ+1cRuiiW2RBMFFRULGisqiiioGKUsvfe2LL0sS1u23L8/5hD3ty67s7Czs7PzeV3XXDOnzJnPEjP3nOec53nM3RERkfhVLtoBREQkulQIRETinAqBiEicUyEQEYlzKgQiInGuQrQDFFW9evW8ZcuW0Y4hIhJTZs2atd3d6+e3LeYKQcuWLUlOTo52DBGRmGJm6460TU1DIiJxToVARCTOqRCIiMQ5FQIRkTinQiAiEuciVgjM7BUz22pmC4+w3czsH2a20szmm1mvSGUREZEji+QZwavAoAK2nwO0Cx4jgH9HMIuIiBxBxPoRuPt3ZtaygF0uBF730DjY08ystpk1dvdNkcokIhJLcnKclF0HWLJ5D0s3pXN6pwZ0bVqr2D8nmh3KmgIbci2nBOt+VgjMbAShswYSExNLJJyISEnaczCTZZvTWbppD0uC52Wb09l3KBsAMziueqUyVwgsn3X5zpLj7qOAUQBJSUmaSUdEYlZ2jrNm+z6WBr/yl27ew5JN6WzcfeB/+9SqUpGOjWpwaVJzOjaqQcfGNWnfsDpVK0XmKzuahSAFaJ5ruRmQGqUsIiIRsTcji+9XbOPb5dtYlBr6lZ+RlQNA+XJG63rV6N2iDlf1T6RTo5p0bFyDRjUTMMvvt3JkRLMQjAduMrNxQD8gTdcHRKQs2LBzP5OXbuWrJVuYvnonh7JzqJFQge7NanN1/xZ0bFyTjo1q0LZBdRIqlo923MgVAjN7GzgVqGdmKcADQEUAd38BmAD8ElgJ7AeGRCqLiEgkZec4czfs4uslW/l6yVaWbUkHoHW9alx7YgtO79SQ3i3qULF86ey6Fcm7hq4oZLsDN0bq80VEIin9YCZTV2zn6yVbmbJsKzv3HaJ8OaNPyzrce24nBnZsQOv61aMdMywxNwy1iEi0bNi5n6+WbGHy0q1MW72DzGynVpWKnNahPgM7NeSU9vWpVaVitGMWmQqBiEgB0vZn8t7sFP6TvIGlm0NNPm3qV2PogFYM7NiA3i3qUKGUNvmES4VARCQf8zbs5o1p6/hkfioHM3Po0bw2953XmdM7NqBlvWrRjlesVAhERAIHDmUzft5G3pi2ngUb06haqTwX9WzGVf0SI9KRq7RQIRCRuLdy617emLaO92enkH4wi3YNqvPQBV24qFdTaibEXpt/UakQiEhcyszO4ctFWxg7bS3TVu+kYnljUNfGDO6XSN9WdUu0Q1e0qRCISFxJ3X2At2esZ9zMDWxLz6Bp7SrceXYHfpPUnPo1Kkc7XlSoEIhImZeT40xduZ2xP65j8tItOHBahwYM7p/IKe0bUL5c/Pz6z48KgYiUWRlZ2Xw8J5XRU1ezYutejqtWiZGntOHKvok0r1s12vFKDRUCESlzdu8/xJvT1/PqD2vZlp5Bx0Y1ePLS7pzXvTGVK0R/bJ/SRoVARMqMDTv38/L3a3g3eQP7D2Xzi3b1eOo33Tmpbb24uvhbVCoEIhLz5m3Yzaipq/l8wSbKmXFBjyYM/0VrOjWuGe1oMUGFQERiUk6OM3npVkZNXc2MNTupUbkCw09uzXUntqRxrSrRjhdTVAhEJKYczMzmwzkbGT11Nau37aNJrQTuPbcTl/VpTo046PwVCSoEIhITdu07xNhp63j9x7Vs33uILk1q8uzlPfhlt8aldpz/WKFCICKl2qa0A4z+bg1vz1jPgcxsTu1QnxEnt+aE1sfpAnAxUSEQkVJp3Y59vPDtKt6blUKOw4U9mjDy5DZ0aFQj2tHKnEILgZm1B/4NNHT3rmZ2PHCBu/854ulEJO4s35LOv6asZPy8VCqUL8dlfZoz8uQ26gAWQeGcEYwG7gReBHD3+Wb2FqBCICLFZkFKGs9NWcHERVuoWqk8w05qxfBftKZBzYRoRyvzwikEVd19Rp62uKwI5RGRODNjzU6em7KS75Zvo2ZCBW4Z2JYhA1pRp1qlaEeLG+EUgu1m1gZwADO7BNgU0VQiUqa5O98u38bzU1Yyc+0u6lWvxF2DOjK4f6JuAY2CcArBjcAooKOZbQTWAIMjmkpEyqScHOfLxZt5fsoqFmxMo3GtBB48vzOX9UmkSiWNARQthRYCd18NnGFm1YBy7p4e+VgiUpZkZefwyfxU/jVlFSu27qXlcVV57NfduKhnMypVUB+AaAvnrqG/Ao+7++5guQ7we3e/N9LhRCS2Zec4n85P5dmvVrB6+z46NKzBs5f34NxujamgTmClRjhNQ+e4+x8PL7j7LjP7JaBCICL5yslxPl+4mWe+Ws6KrXvp2KgGLwzuxVmdG1EuzieBKY3CKQTlzayyu2cAmFkVID7ncxORArk7Exdt4ZmvlrN0czptG1Tn+St7cU5XFYDSLJxC8AbwtZmNIXTn0FDgtYimEpGY4h4aCfSpSctZlLqH1vWq8ezlPTjv+CZxPw1kLAjnYvHjZrYAOB0w4BF3nxjxZCJS6h2+DfTpScuZl5JGYt2qPHlpdy7s0UTXAGJIWGMNufvnwOcRziIiMcLd+e/KHTw1aRmz1++mae0qPP7r47moV1ONBBqDwrlr6GLgMaABoTMCA9zdNfWPSByatnoHT01azow1O2lcK4G/XNSVS3s3122gMSycM4LHgfPdfUmkw4hI6ZW8didPTVrOD6t20KBGZR66oAuX922uyeDLgHAKwRYVAZH4tXxLOo99vpSvl26lXvVK3HdeZ67ql0hCRRWAsiKcQpBsZu8AHwEZh1e6+wcRSyUiUbcp7QBPT1rOe7NSqFapAv83qAPXndiSqpU0jUlZE87/ojWB/cBZudY5oEIgUgalHcjkhW9X8cr3a3CHIQNacdNpbTUaaBkWzu2jQ0oiiIhEV0ZWNmN/XMdzU1aye38mv+rRhN+f1UETwsSBcO4aSgCGAV2A/80Q4e5DI5hLREpITo4zfl4qf/9yGSm7DvCLdvW4a1BHujatFe1oUkLCud9rLNAIOBv4FmgGhDUCqZkNMrNlZrbSzO7OZ3uimU0xszlmNj8Yw0hESsjUFds4/7nvue2dudRMqMjYYX0ZO6yfikCcCecaQVt3v9TMLnT314JpKgvtWWxm5YHngTOBFGCmmY1398W5drsXeNfd/21mnYEJQMsi/xUiUiQLN6bx2BdLmbpiO83qVOGZy3pwQfcmGg8oToVTCDKD591m1hXYTHhf1n2BlcF8BpjZOOBCIHchcEIXowFqAalhHFdEjtKGnft58stlfDQ3ldpVK3LvuZ24+oQW6gsQ58IpBKOCOQjuA8YD1YH7w3hfU2BDruUUoF+efR4EvjSzm4FqwBn5HcjMRgAjABITE8P4aBHJLW1/Jv+YvIKxP67DDG44tQ2/PaUNtapoWkgJ766hl4KX3wKti3Ds/M4xPc/yFcCr7v6kmZ0AjDWzru6ekyfDKELTZZKUlJT3GCJyBFnZObw9Yz1PTVpO2oFMLundjNvPbE/jWlWiHU1KkSMWAjMb7O5vmNkd+W1396cKOXYK0DzXcjN+3vQzDBgUHO/H4A6lesDWwoKLSMH+u3I7D3+ymGVb0unfui73n9eFzk00RJj8XEFnBNWC5xpHeeyZQDszawVsBC4Hrsyzz3pCw1u/amadCN2euu0oP09EgHU79vGXz5bw5eItNKtThRcG9+LsLo0w04Vgyd8RC4G7vxjc+bPH3Z8u6oHdPcvMbiJ0h1F54BV3X2RmDwPJ7j4e+D0w2sxuJ9RsdJ27q+lH5CjszcjiuckreeX7NVQob9x5dgeGndRKYwJJoayw710zm+Lup5VQnkIlJSV5cnJytGOIlBo5Oc77s1N4fOIytqVncHGvptw1qCMNayYU/maJG2Y2y92T8tsWzl1DP5jZc8A7wL7DK919djHlE5GjNGvdTh76ZDHzU9LomVib0dck0aN57WjHkhgTTiE4MXh+ONc6BwYWfxwRCUfq7gM8+vlSxs9LpVHNBJ65rAcX9mii6wByVMK5fbTUNAuJxLsDh7IZ9d1q/v3tStzhloFt+e2pbTQ0tByTsP7rMbNz+fmgcw8f+R0iUpzcnU/nb+JvE5aQmnaQc49vzD3ndKRZHY0MKscunNFHXwCqAqcBLwGXADMinEtEAss2p3P/xwuZvmYnXZrU5OnLetCv9XHRjiVlSFjXCNz9eDOb7+4PmdmTaFIakYjbczCTZyat4LUf11IjoQJ/uagrl/dJpLwGhpNiFk4hOBA87zezJsAOoFXkIonEN3fno7kb+ctnS9mxL4Mr+iZy51kdNEOYREw4heBTM6sNPAHMJnTH0OiIphKJU4tT9/DA+IXMXLuL7s1r88p1SRzfTLeDSmQVNNZQRXfPdPdHglXvm9mnQIK7p5VMPJH4kHYgk6cnLef1H9dSu2olHvt1Ny7t3VzzA0iJKOiMYKOZfQy8DUzxkAwgo2SiiZR9OTnOB3M28ujnS9ix7xCD+7Xg92e1p3ZVNQNJySmoEHQidIfQfcDrZvYe8La7Ty+RZCJl3KLUNO7/eBGz1u2iZ2JtXh3SV1NESlQUNOjcDuBF4MXgIvGlwDNm1gAY5+5/KqGMImVK2v5Mnpy0jDemraNO1Uo8fsnxXNKrmZqBJGrC6lDm7qlm9jKwC7gDuB5QIRApgpwc571ZKTz6xVJ27z/E1f1bcMeZHahVVbOESXQVWAiCiWLOJzST2ADgC+Ae4MvIRxMpOxZuTOO+jxcyZ/1uklrU4aEL+9KliZqBpHQo6K6htwjNIfwd8BZwpbsfLKlgImXBvowsnpq0nDH/XUPdapV48tLuXNyrqQaHk1KloDOCicBId08vqTAiZcnERZt5cPwiNqUd5Mp+idx1dkc1A0mpVNDF4tdKMohIWbFx9wEe+HgRXy3ZQsdGNXjuyl70blEn2rFEjkhj14oUk6zsHMb8dy1Pf7Ucd7jnnI4MPakVFcuXi3Y0kQKpEIgUgznrd/HHDxeyZNMeTu/YgAcv6ELzuhoiWmJDQReLLy7oje6uEUgl7qUdyOSJiUt5c/p6GtZI4IXBvTi7SyNdDJaYUtAZwfnBcwNC01VODpZPA75BQ1FLHHN3Ppm/iUc+XcyOvRlcd2JLfn9WB6pX1km2xJ6CLhYPAQgGmuvs7puC5cbA8yUTT6T0WbdjH/d+tJCpK7bTrWktXrm2D92aqU+AxK5wfr60PFwEAluA9hHKI1JqHcrKYdR3q/jn5JVULF+OB8/vzNUntNREMRLzwikE35jZREKjkDpwOTAloqlESpmZa3dyzwcLWLl1L7/s1oj7z+tCo1oJhb9RJAYUWgjc/SYzuwg4OVg1yt0/jGwskdJhz8FMHv9iKW9MW0/T2lUYc10fTuvYINqxRIpVuFe2ZgPp7v6VmVU1sxrqcSxl3aTFW7jvo4VsTT/IsJNacceZ7ammi8FSBhX6X7WZDQdGAHWBNkBT4AXg9MhGE4mOrekHeWj8Yj5bsImOjWrwwtW96dFc00VK2RXOz5sbgb7AdAB3XxHMSSBSprg7/0lO4c+fLeZgVg53nt2BESe3Vs9gKfPCKQQZ7n7ocAcZM6tA6KKxSJmxdvs+7vlgAT+u3kHfVnX528XdaFO/erRjiZSIcArBt2b2R6CKmZ0J/A74JLKxREpGVnYOo6eu4ZmvllOpQjn+dnE3LkvSpPESX8IpBHcDw4AFwEhgAvBSJEOJlISFG9O46/35LErdw9ldGvLwhV1pWFO3hEr8Cef20RxgdPAQiXkHDmXz9FfLeWnqaupVr8wLg3sxqGvjaMcSiZpw7hoaADwItAj2N8DdvXVko4kUv+9XbOePHy5g/c79XNE3kbvP6UitKposRuJbOE1DLwO3A7OA7MjGEYmMtP2ZPPLZYt6blUKretUYN6I//VsfF+1YIqVCOIUgzd0/j3gSkQj5eskW7vlgATv3HeLG09pw88B2JFQsH+1YIqVGOIVgipk9QWjY6YzDK919dsRSiRSDtAOZPPzJYt6fnULHRjV45bo+dG2qUUJF8gqnEPQLnpNyrXNgYGFvNLNBwLNAeeAld380n31+Q+gahAPz3P3KMDKJFGjKsq3c8/4Ctu3N4OaBbbl5YDsqVVDHMJH8hHPX0GlHc2AzK09o3oIzgRRgppmNd/fFufZpB9wDDHD3XeqxLMdqz8FM/vzpYt5NTqF9w+qMuqY3xzfT8BAiBSloqsrB7v6Gmd2R33Z3f6qQY/cFVrr76uB444ALgcW59hkOPO/uu4Jjbi1KeJHcvlu+jbven8+WPQf53altuPWMdlSuoGsBIoUp6IygWvBc4yiP3RTYkGs5hZ+amQ5rD2Bm/yXUfPSgu3+R90BmNoLQwHckJiYeZRwpq9IPZvLXCUt4e8YG2jaozge/G6BB4kSKoKCpKl8Mnh86ymPn10c/7xhFFYB2wKlAM2CqmXV19915sowCRgEkJSVpnCP5n+9XbOeu9+ezKe0AI09pze1ntNcdQSJFFE6HsgRCQ0x0Af7X/97dhxby1hSgea7lZkBqPvtMc/dMYI2ZLSNUGGYWHl3i2d6MLP42YQlvTl9P6/rVeO+GE+mVWCfasURiUji3UYwFGgFnA98S+kIPZ1KamUA7M2tlZpUITXE5Ps8+HwGnAZhZPUJNRavDiy7x6odV2xn0zHe8NWM9w3/Rigm3/EJFQOQYhHP7aFt3v9TMLnT318zsLWBiYW9y9ywzuynYtzzwirsvMrOHgWR3Hx9sO8vMFhPqtXynu+84+j9HyrL9h7J47POlvPbjOlrVq8Z/Rp5AUsu60Y4lEvPCKQSZwfNuM+sKbAZahnNwd59AaLTS3Ovuz/XagTuCh8gRzVq3k9vfmceGXfsZOqAVd57dgSqVdC1ApDiEUwhGmVkd4D5CTTvVgfsLfotI8cjKzuH5Kav4x+QVNKmdwLjh/emnMYJEilU4HcoOzz3wLaARR6XEbNx9gNvGzWHm2l1c1LMpD1/YhRoJGilUpLgV1KGswOaaMDqUiRy1z+Zv4p4P5pPj8PRl3bmoZ7NoRxIpswo6IzjajmQiR21fRhYPfbKId5NT6NG8Ns9e3oMWx1Ur/I0ictQK6lB2tB3JRI7Kwo1p3PL2HNbs2MdNp7Xl1jPaUbG8BooTibRwOpS1JjSCaH9CPYN/BG4/PIaQyLHKyXFe+n41T0xcRr3qlXl7uCaNESlJ4dw19BahUUQvCpYvB97m5+MGiRTZ1j0H+f1/5jF1xXYGdWnEo7/uRu2qlaIdSySuhFMIzN3H5lp+I+goJnJMvl6yhTvfm8/+Q1n87eJuXN6nOWb5DVElIpEU7gxldwPjCDUNXQZ8ZmZ1Adx9ZwTzSRl0MDObv01Ywms/rqNz45r844qetG1QPdqxROJWOIXgsuB5ZJ71QwkVBvUtkLAt25zOLW/PYdmWdK4/qRV3DuqgOQNEoiycDmWtSiKIlG3uzhvT1vHnz5ZQI6ECrw7pw6kdNCGdSGlQ6L15ZvZIMO3k4eWaZjYmsrGkLNmXkcVNb8/hvo8XcUKb4/j81pNVBERKkXCahioAM8xsCKHhqP8ZPEQKtXb7PkaOncWKrencfU5HRp7cWheERUqZcJqG7jGzr4HpwC7gZHdfGfFkEvMmL93CrePmUr6c8frQfpzUrl60I4lIPsJpGjqZUIeyh4FvgOfMrEmEc0kMy8lxnv1qBcNeSyaxblU+uekkFQGRUiycpqG/A5e6+2IAM7sYmAx0jGQwiU17DmZyxzvz+GrJFi7u2ZS/XtxNcwiLlHLhFIIT3D378IK7f2Bm30Ywk8SoFVvSGTl2Fut37ufB8ztz7YktdT1AJAYcsWnIzJ4BcPdsM7s1z+YnI5pKYs7nCzbxq+f/y56DWbw1vD/XDWilIiASIwq6RnByrtfX5tl2fASySAzKznEe+2IpN7w5m/aNavDpzSfRt5XmERaJJQU1DdkRXosAsGvfIW4ZN4epK7ZzZb9EHji/s3oJi8SgggpBuWCu4nK5Xh8uCPp/e5xblJrGyLGz2Long0cv7sblfROjHUlEjlJBhaAWMIufvvxn59rmEUskpd5HczZy9wfzqV2lEu+M7E/PxDrRjiQix6CgGcpalmAOiQGZ2Tn8dcISxvx3LX1b1eX5K3tRv0blaMcSkWMUzu2jImzfm8Hv3pzNjDU7GTqgFff8sqOmkRQpI1QIpFALUtIYMTaZnfsO8cxlPfhVz6bRjiQixUiFQAr00ZyN3PX+fOpVr8z7N5xI16a1oh1JRIpZWIXAzE4C2rn7GDOrD1R39zWRjSbRdLh/wKjvVtO3VV3+dVUv6lXX9QCRsqjQQmBmDwBJQAdgDFAReAMYENloEi279x/i5rdD/QOuOaEF953XWdcDRMqwcM4ILgJ6Etw+6u6pZlYjoqkkapZvSWf468mk7j6g/gEicSKcQnDI3d3MHMDMqkU4k0TJxEWbueOduVStXIFxI/rTu4WGihCJB+EUgnfN7EWgtpkNJzRp/ejIxpKSlJPjPPv1Cp79egXdm9XixauTaFQrIdqxRKSEhDND2d/N7ExgD6HrBPe7+6SIJ5MSsTcjizvemcuXi7fw617N+MtFXTV/gEicCedi8e3Af/TlX/as3b6P4a8ns3r7Pu4/rzNDBmj+AJF4FE7TUE1gopntBMYB77n7lsjGkkj7dvk2bn5rNuXKGa8P7cuAtppKUiReFXpPoLs/5O5dgBuBJsC3ZvZVxJNJRLg7o75bxZAxM2hSuwrjbzxJRUAkzhWlZ/FWYDOwA2gQmTgSSQczs7nr/fl8PDeVX3ZrxBOXdKdaZXUuF4l3hZ4RmNkNZvYN8DVQDxju7mHNUGZmg8xsmZmtNLO7C9jvEjNzM0sKN7gUzaa0A1zywg+Mn5fKH85qz/NX9lIREBEgvDOCFsBt7j63KAc2s/LA88CZQAow08zGu/viPPvVAG4Bphfl+BK+pZv3cN0rM9mbkcXoq5M4o3PDaEcSkVKkoMnrawYvHwfWm1nd3I8wjt0XWOnuq939EKELzRfms98jwWccLGJ2CcMPq7Zz6Qs/4jjvjjxBRUBEfqagpqG3gudZQHLwPCvXcmGaAhtyLacE6/7HzHoCzd3904IOZGYjzCzZzJK3bdsWxkcLwPh5qVz3ykwa1kzgg98NoHOTmoW/SUTiTkEzlJ0XPLc6ymPnd0P6/6a4NLNywNPAdYUdyN1HAaMAkpKSNE1mIdyd0VNX89cJS+nbsi6jr0miVtWK0Y4lIqVUOBeLvw5nXT5SgOa5lpsBqbmWawBdgW/MbC3QHxivC8bHJjvHeeiTxfx1wlLO7daY14f1VREQkQId8YzAzBKAqkA9M6vDT7/waxLqT1CYmUA7M2sFbAQuB648vNHd0wjdhXT4874B/uDu4TQ7ST4OZmZz+ztz+XzhZoYOaMW953aiXDn1FBaRghV019BI4DZCX/qz+KkQ7CF0N1CB3D3LzG4CJgLlgVfcfZGZPQwku/v4Y0ou/5/d+w8x/PVkZq7dxb3nduL6X7SOdiQRiRHmXnCTu5nd7O7/LKE8hUpKSvLkZJ005Jayaz/XjZnJ+h37eeqy7px3fDgnbCIST8xslrvn2/Qezuij/zSzrkBnICHX+teLL6IcrUWpaQwZM5ODmdm8Pqwv/VsfF+1IIhJjwp2q8lRChWACcA7wPaBCEGVTV2zjhjdmUyOhAu/dcCLtG2riOBEpunAmor0EOB3Y7O5DgO6AZjGPsg9mpzBkzEya1anCh78boCIgIkctnCEmDrh7jpllBb2NtwK6Ehkl7s6/vlnFExOXcWKb43jh6t7UTNDtoSJy9MIpBMlmVpvQ9JSzgL3AjIimknxl5zgPjF/IG9PWc2GPJjxxSXcqVQjnpE5E5MjCuVj8u+DlC2b2BVDT3edHNpbkdeBQNreMm8OkxVv47Slt+L+zO6iPgIgUi4I6lPUqaJu7z45MJMkr/WAmw15NZua6nTx0QReuPbFltCOJSBlS0BnBkwVsc2BgMWeRfOzad4hrx8xgceoe/nF5T87vrj4CIlK8Chp07rSSDCI/t3XPQQa/PJ21O/Yz6preDOyoIaRFpPiF04/gmvzWq0NZZG3YuZ/BL09ne3oGrw7pw4ltNK+wiERGOHcN9cn1OoFQn4LZqENZxKzcupfBL03nQGY2b1zfj56JdaIdSUTKsHDuGro597KZ1QLGRixRnFu4MY1rX5mBmTFuRH86NdZkMiISWUdzE/p+oF1xBxGYtW4nV4yeRuUK5Xh3pIqAiJSMcK4RfMJPM4uVIzTm0LuRDBWPvl+xneGvJ9OoVgJvXN+PprWrRDuSiMSJcK4R/D3X6yxgnbunRChPXJq0eAs3vjmb1vWrMXZYP+rX0FBOIlJywrlG8C1AMM5QheB1XXffGeFsceHjuRu54915dGtai1eH9KF21UrRjiQicSacpqERwCPAASCH0ExljgaeO2ZvTV/Pnz5aQL9WdXnp2j5UrxzOCZqISPEK55vnTqCLu2+PdJh4Muq7Vfx1wlIGdmzAv67qRULF8tGOJCJxKpxCsIrQnUJSDNydpyct5x+TV3Lu8Y15+jc9NIKoiERVOIXgHuAHM5sOZBxe6e63RCxVGZWT4zzy2WLG/HctlyU1568Xd6O8RhAVkSgLpxC8CEwGFhC6RiBHITvHueeD+bybnMLQAa2477xOmKkIiEj0hVMIstz9jognKcNycpw735vHB7M3cuvp7bjtjHYqAiJSaoTTOD3FzEaYWWMzq3v4EfFkZYS7c//4hXwweyN3nNme289sryIgIqVKOGcEVwbP9+Rap9tHw+DuPPrFUt6Ytp6Rp7Tm5oFtox1JRORnwulQ1qokgpRFz01eyYvfrubq/i24e1BHnQmISKmk+Qgi5OXv1/DkpOVc3KspD13QRUVAREotzUcQAW/PWM8jny7mnK6NePzXx2uSeREp1TQfQTH7eO5G/vjhAk7tUGOM3EoAAAytSURBVJ9nL+9JhfLqLCYipZvmIyhGXy7azB3vzqNfq7q8MLi3egyLSEzQfATFZOqKbdz01hy6Na3FS9f20dhBIhIzNB9BMZi5difDX0+mTYPqvDakr0YRFZGYcsRvLDNrCzQ8PB9BrvW/MLPK7r4q4uliwPyU3QwdM5MmtaswdlhfalWtGO1IIiJFUlAj9jNAej7rDwTb4t6yzelc88oMalWtyJvX96Nedc0sJiKxp6BC0NLd5+dd6e7JQMuIJYoRa7bvY/DL06lcoRxvXd+fxrU0x7CIxKaCCkFCAdvi+ltv4+4DXDV6Gtk5zpvX9yPxuKrRjiQictQKKgQzzWx43pVmNgyYFc7BzWyQmS0zs5Vmdnc+2+8ws8VmNt/MvjazFuFHj46tew5y1ehppGdk8frQvrRtUCPakUREjklBt7fcBnxoZlfx0xd/ElAJuKiwA5tZeeB54EwghVBhGe/ui3PtNgdIcvf9ZnYD8DhwWdH/jJKxa98hBr88na3pGYwd1o+uTWtFO5KIyDE7YiFw9y3AiWZ2GtA1WP2Zu08O89h9gZXuvhrAzMYBFwL/KwTuPiXX/tOAwUXIXqLSD2Zy7ZgZrN2xn1ev60PvFnWiHUlEpFiEM8TEFGBKYfvloymwIddyCtCvgP2HAZ8fxedEXGZ2Dje8MZvFqXsYdU1vTmxbL9qRRESKTSR7PuU30prnsw4zG0yo2emUI2wfAYwASExMLK58YXF3/vThAr5fuZ2/X9qdgR0blujni4hEWiQHw0kBmudabgak5t3JzM4A/gRc4O4Z+R3I3Ue5e5K7J9WvXz8iYY/k+SkreTc5hVtOb8clvZuV6GeLiJSESBaCmUA7M2tlZpWAy4HxuXcws57Ai4SKwNYIZjkqH83ZyN+/XM7FPZty+xkaZ09EyqaIFQJ3zwJuAiYCS4B33X2RmT1sZhcEuz0BVAf+Y2ZzzWz8EQ5X4qat3sH/vTef/q3r8uivj9fEMiJSZkV0dDR3nwBMyLPu/lyvz4jk5x+tVdv2MnLsLJrXrcKLg5M0nLSIlGn6hstj+94MhoyZScXyxqtDNIiciJR9Gi85l4OZ2Vz/WjJb0w8ybsQJNK+roSNEpOxTIQjk5Di3jZvLvJTd/Puq3vRoXjvakURESoSahgJ/+3wJXyzazL3ndmZQ10bRjiMiUmJUCICxP65l9NQ1XHdiS4YOaBntOCIiJSruC8HkpVt4YPwizujUgPvO66zbREUk7sR1IVi4MY2b3ppDlya1+McVPSlfTkVAROJP3BaC1N0HGPrqTOpUrcTL1yZRtZKum4tIfIrLQrDnYCZDxszkwKFsxgzpQ4OaBU3GJiJStsXdz+DM7BxufHM2q7bt5bWhfWnfUDOMiUh8i6tC4O7c++FCpq7YzuOXHM8AzSsgIhJfTUP/+mYV7yRv4JaBbflNUvPC3yAiEgfiphB8PHcjT0xcxkU9m3L7me2jHUdEpNSIm0LQoEYCZ3VuyKO/7qa+AiIiucTNNYIT2hzHCW2Oi3YMEZFSJ27OCEREJH8qBCIicU6FQEQkzqkQiIjEORUCEZE4p0IgIhLnVAhEROKcCoGISJwzd492hiIxs23AumjnOEr1gO3RDnEMlD/6Yv1vUP7oaeHu9fPbEHOFIJaZWbK7J0U7x9FS/uiL9b9B+UsnNQ2JiMQ5FQIRkTinQlCyRkU7wDFS/uiL9b9B+UshXSMQEYlzOiMQEYlzKgQiInFOhaAEmFmCmc0ws3lmtsjMHop2pqIys9pm9p6ZLTWzJWZ2QrQzFYWZ3WpmC4N//9uinacwZvaKmW01s4W51j0R/PvPN7MPzax2NDMW5Aj5HzSzjWY2N3j8MpoZC3KE/D3MbFqQPdnM+kYzY3FSISgZGcBAd+8O9AAGmVn/KGcqqmeBL9y9I9AdWBLlPGEzs67AcKAvoeznmVm76KYq1KvAoDzrJgFd3f14YDlwT0mHKoJX+Xl+gKfdvUfwmFDCmYriVX6e/3HgIXfvAdwfLJcJKgQlwEP2BosVg0fMXKU3s5rAycDLAO5+yN13RzdVkXQCprn7fnfPAr4FLopypgK5+3fAzjzrvgzyA0wDmpV4sDDllz+WHCG/AzWD17WA1BINFUEqBCXEzMqb2VxgKzDJ3adHO1MRtAa2AWPMbI6ZvWRm1aIdqggWAieb2XFmVhX4JdA8ypmO1VDg82iHOAo3BU1br5hZnWiHKaLbgCfMbAPwd0r3GVmRqBCUEHfPDk4pmwF9g+aKWFEB6AX82917AvuAu6MbKXzuvgR4jFDTyhfAPCCrwDeVYmb2J0L534x2liL6N9CGUPPoJuDJ6MYpshuA2929OXA7wRlyWaBCUMKCJpVvyL/9tLRKAVJyncW8R6gwxAx3f9nde7n7yYRO+VdEO9PRMLNrgfOAqzzGOgG5+5bgB1EOMJrQNZtYci3wQfD6P8Re/iNSISgBZlb/8B0eZlYFOANYGt1U4XP3zcAGM+sQrDodWBzFSEVmZg2C50TgYuDt6CYqOjMbBNwFXODu+6Odp6jMrHGuxYsINdnFklTglOD1QGL0x0R+KkQ7QJxoDLxmZuUJFd933f3TKGcqqpuBN82sErAaGBLlPEX1vpkdB2QCN7r7rmgHKoiZvQ2cCtQzsxTgAUJt0pWBSWYGoQvgv41ayAIcIf+pZtaD0EXXtcDIqAUsxBHyDweeNbMKwEFgRPQSFi8NMSEiEufUNCQiEudUCERE4pwKgYhInFMhEBGJcyoEIiJxToVAIs7M3MyezLX8BzN7sJiO/aqZXVIcxyrkcy4NRl2dks+29mY2wcxWBvu8a2YNI50pkszsV2bWOdo5pGSoEEhJyAAuNrN60Q6SW9CvI1zDgN+5+2l5jpEAfEZo+I227t6J0FAK9YsvaVT8ClAhiBMqBFISsgjN9Xp73g15f9Gb2d7g+VQz+zb4db3czB41s6uCeR0WmFmbXIc5w8ymBvudF7y/fDB+/8xgkLORuY47xczeAhbkk+eK4PgLzeyxYN39wEnAC2b2RJ63XAn86O6fHF7h7lPcfWEwD8WY4HhzzOy04HjXmdlHZvaJma0xs5vM7I5gn2lmVjfY7xsze8bMfgjy9A3W1w3ePz/Y//hg/YPBYG7fmNlqM7sl1981OPi3m2tmLx4ugma218z+YqG5MqaZWUMzOxG4gNAAa3PNrI2Z3WJmi4PPHBfO/+gSQ9xdDz0i+gD2Ehq+dy2h4Xv/ADwYbHsVuCT3vsHzqcBuQr2yKwMbCY0FD3Ar8Eyu939B6EdNO0LjIiUQ6vV5b7BPZSAZaBUcdx/QKp+cTYD1hH7NVwAmA78Ktn0DJOXznqeAW4/wd/8eGBO87hgcOwG4DlgJ1Ag+Kw34bbDf08BtuT5zdPD6ZGBh8PqfwAPB64HA3OD1g8APwd9bD9hBaMjzTsAnQMVgv38B1wSvHTg/eP14rn+zvP+7pAKVg9e1o/3flB7F+9AZgZQId98DvA7cUti+ucx0903ungGsAr4M1i8AWuba7113z3H3FYSGv+gInAVcY6Ghv6cDxxEqFAAz3H1NPp/XB/jG3bd5aNz/Nwl9AR+tk4CxAO6+FFgHtA+2TXH3dHffRqgQHD6jyPu3vR28/zugZjBmVe7jTgaOM7Nawf6fuXuGu28nNOR5Q0JjQ/UGZgb/HqcTGloc4BBweLiTWXk+O7f5hIYYGUwMj9wq+dNYQ1KSngFmA2NyrcsiaKK00AA6lXJty8j1OifXcg7//3+7ecdJccCAm919Yu4NZnYqoTOC/Fihf8HPLeKngciKcrxj/dvyOrxf7uNmB8cy4DV3z2/8/Ex39zz75+dcQkXxAuA+M+viP02SIzFOZwRSYtx9J/AuoQuvh60l9GsV4EJCTRlFdamZlQuuG7QGlgETgRvMrCL8786ewibTmQ6cYmb1gjb0KwjNZlaQt4ATzezcwyvMbJCZdQO+A646/PlAYpCtKC4L3n8SkObuaXmOeyqwPTjjOpKvgUvspxFY65pZi0I+N51Q0xVmVg5o7u5TgP8DagPVi/h3SCmmMwIpaU8CN+VaHg18bGYzCH1hHenXekGWEfrCbkiorf2gmb1EqJljdnCmsY3QnTBH5O6bzOweYAqhX9ET3P3jQt5zILhA/YyZPUNodNP5hK5j/IvQBeYFhM58rnP3jFCcsO0ysx8IXWMZGqx7kNBscfOB/YTGyS8o42Izuxf4MvhSzwRuJNRUdSTjgNHBBefLgZeD5icjNO9wLE1VKoXQ6KMipZSZfQP8wd2To51FyjY1DYmIxDmdEYiIxDmdEYiIxDkVAhGROKdCICIS51QIRETinAqBiEic+39OfubF7PLmqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "\n",
    "\n",
    "ax = plt.figure().gca()\n",
    "ax.plot(PCA_feature_num_list, PCA_explainedVariances_list)\n",
    "ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n",
    "plt.xlabel('Number of Components')\n",
    "plt.ylabel('Cumulative Explained Variance');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's pretty apparent that ALL the features should be used in making prediction, as each component noteciably adds variance in relation to the total variance - almost in a straight-line fashion.\n",
    "\n",
    "\n",
    "## Model Comparison##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.tuning import ParamGridBuilder\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.classification import GBTClassifier\n",
    "from pyspark.ml.classification import LinearSVC\n",
    "from pyspark.ml.classification import OneVsRest\n",
    "from pyspark.ml.classification import NaiveBayes\n",
    "\n",
    "\n",
    "PARAMETER_CHOICE_NUM = 1 # Just for practice\n",
    "classification_model_list = []\n",
    "\n",
    "LR = LogisticRegression(labelCol=LABEL_COL_NAME, featuresCol=STANDARDIZED_FEATURES_COL_NAME)\n",
    "LR_paramGrid = (ParamGridBuilder()\n",
    "             .addGrid(LR.regParam, np.linspace(0.01, 2, PARAMETER_CHOICE_NUM))\n",
    "             .addGrid(LR.elasticNetParam, np.linspace(0, 1, PARAMETER_CHOICE_NUM))\n",
    "             .addGrid(LR.maxIter, np.linspace(1, MAX_ITER_NUM, PARAMETER_CHOICE_NUM, dtype = int))\n",
    "             .build())\n",
    "classification_model_list.append([LR, LR_paramGrid])\n",
    "\n",
    "\n",
    "RF = RandomForestClassifier(labelCol=LABEL_COL_NAME, featuresCol=STANDARDIZED_FEATURES_COL_NAME)\n",
    "RF_ParamGrid = (ParamGridBuilder()\n",
    "             .addGrid(RF.numTrees, np.linspace(1, MAX_ITER_NUM, PARAMETER_CHOICE_NUM, dtype = int))\n",
    "             .addGrid(RF.maxDepth, np.linspace(1, COL_TOTAL_NUM, PARAMETER_CHOICE_NUM))\n",
    "             .build())\n",
    "classification_model_list.append([RF, RF_ParamGrid])\n",
    "\n",
    "\n",
    "GB = GBTClassifier(labelCol=LABEL_COL_NAME, featuresCol=STANDARDIZED_FEATURES_COL_NAME)\n",
    "GB_ParamGrid = (ParamGridBuilder()\n",
    "             .addGrid(GB.maxIter, np.linspace(1, MAX_ITER_NUM, PARAMETER_CHOICE_NUM, dtype = int))\n",
    "             .addGrid(GB.maxDepth, np.linspace(1, COL_TOTAL_NUM, PARAMETER_CHOICE_NUM))\n",
    "             .build())\n",
    "classification_model_list.append([GB, GB_ParamGrid])\n",
    "\n",
    "\n",
    "LSVC = LinearSVC(maxIter=MAX_ITER_NUM)\n",
    "LSVC_paramGrid = (ParamGridBuilder()\n",
    "             .addGrid(LSVC.regParam, np.linspace(0.01, 2, PARAMETER_CHOICE_NUM))\n",
    "             .addGrid(LSVC.maxIter, np.linspace(1, MAX_ITER_NUM, PARAMETER_CHOICE_NUM, dtype = int))\n",
    "             .build())\n",
    "classification_model_list.append([LSVC, LSVC_paramGrid])\n",
    "\n",
    "\n",
    "# OVR = OneVsRest(classifier=LR)\n",
    "# OVR_paramGrid = (ParamGridBuilder().build())\n",
    "# classification_model_and_paramgrid_list.append([OVR, OVR_paramGrid])\n",
    "# IllegalArgumentException: 'Field \"rawPrediction\" does not exist.\\nAvailable fields: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 1_INDEX, 1_VEC, 3_INDEX, 3_VEC, 4_INDEX, 4_VEC, 6_INDEX, 6_VEC, 7_INDEX, 7_VEC, 9_INDEX, 9_VEC, 10_INDEX, 10_VEC, 12_INDEX, 12_VEC, 14_INDEX, 14_VEC, 15_INDEX, 15_VEC, 17_INDEX, 17_VEC, 19_INDEX, 19_VEC, 20_INDEX, 20_VEC, label, features, std_features, pca_features, CrossValidator_01aa8f348031_rand, prediction'    \n",
    "\n",
    "NB = NaiveBayes()\n",
    "NB_paramGrid = (ParamGridBuilder()\n",
    "             .addGrid(NB.smoothing, np.linspace(0, 1, PARAMETER_CHOICE_NUM))\n",
    "             .build())\n",
    "classification_model_list.append([NB, NB_paramGrid])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Wei-Ming Lee](https://www.wiley.com/en-ca/Python+Machine+Learning-p-9781119545637) states, \"The purpose of cross-validation is not for training your model, but rather it is for model checking. Cross-validation (CV) is useful when you need to compare different machine learning algorithms to see how they perform with the given dataset.\"\n",
    "\n",
    "As per above, a sample, canonical set of parameters of each model, cited in Spark doc, are to be tested in much fewer permutations via `ParamGrid` as part of CV, in order to determine the best `areaUnderROC` of each model.\n",
    "\n",
    "Although `ParamGrid` is tantamount to scikit-learn's [GridSearchCV](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html), it is strictly speaking used for hypermarameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.tuning import CrossValidator\n",
    "\n",
    "\n",
    "evaluator = BinaryClassificationEvaluator()\n",
    "\n",
    "for classification_model in classification_model_list:\n",
    "    cv = CrossValidator(estimator=classification_model[0], \n",
    "                        estimatorParamMaps=classification_model[1],\n",
    "                        evaluator=evaluator,\n",
    "                        numFolds=5)\n",
    "\n",
    "    cvModel = cv.fit(df_train)\n",
    "    predictions = cvModel.transform(df_test)\n",
    "    \n",
    "    model_details_in_list = classification_model_list[classification_model_list.index(classification_model)]\n",
    "    model_details_in_list.append(evaluator.evaluate(predictions)) # By default, areaUnderROC\n",
    "    model_details_in_list.append(cvModel.bestModel)\n",
    "#     print(evaluator.evaluate(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression: 0.7211899489746785\n",
      "GBTClassifier: 0.681188023490902\n",
      "RandomForestClassifier: 0.5\n",
      "LinearSVC: 0.4781457591219794\n",
      "NaiveBayes: 0.40627707711562505\n"
     ]
    }
   ],
   "source": [
    "# classification_model_and_paramgrid_list[x][0] = an instannce of classification model\n",
    "# classification_model_and_paramgrid_list[x][1] = ParamGrid list\n",
    "# classification_model_and_paramgrid_list[x][2] = The default 'areaUnderROC' value of the best model of the classification model according to CV\n",
    "# classification_model_and_paramgrid_list[x][3] = The best model\n",
    "\n",
    "# Sort by 'areaUnderROC' value of each classification model\n",
    "classification_model_list.sort(key=lambda model: model[2], reverse=True)\n",
    "\n",
    "for classification_model in classification_model_list:\n",
    "    model_name = str(classification_model[0]) # classification_model[0] is the best model, when sorted\n",
    "    print(model_name[:model_name.find(\"_\")] + \": \" + str(classification_model[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('2', 0.4425948084598169),\n",
       " ('1_INDEX', 0.39618089494649256),\n",
       " ('5', 0.2921000749056233),\n",
       " ('15_INDEX', 0.27947773614577814),\n",
       " ('12_INDEX', 0.2298915437703011),\n",
       " ('3_INDEX', 0.22041131780203232),\n",
       " ('14_INDEX', 0.19825481712899),\n",
       " ('4_INDEX', 0.1947921605633574),\n",
       " ('9_INDEX', 0.1782953512786073),\n",
       " ('8', 0.1680830998972306),\n",
       " ('7_INDEX', 0.1644038874121714),\n",
       " ('10_INDEX', 0.12518506615774153),\n",
       " ('17_INDEX', 0.031353772949419816),\n",
       " ('11', 0.014629371319035049),\n",
       " ('18', -0.026711081145265846),\n",
       " ('16', -0.06291397012281359),\n",
       " ('19_INDEX', -0.12799455889359893),\n",
       " ('13', -0.1768630476426147),\n",
       " ('20_INDEX', -0.17989922498426686),\n",
       " ('6_INDEX', -0.304823570093416)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from itertools import chain\n",
    "\n",
    "\n",
    "best_model = classification_model_list[0][3]\n",
    "\n",
    "final_model = best_model.transform(df_pipeline)\n",
    "attrs = sorted((attr[\"idx\"], attr[\"name\"]) for attr in (chain(*final_model\n",
    "        .schema['features'].metadata[\"ml_attr\"][\"attrs\"].values())))\n",
    "\n",
    "sorted([(name, best_model.coefficients[idx]) for idx, name in attrs], key=lambda coefficient: coefficient[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = \"./model\"\n",
    "classification_model_list[0][3].save(model_path)\n",
    "\n",
    "# model_loaded = RandomForestClassificationModel.load(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Addendum #\n",
    "\n",
    "## **`dropLast=False`** in OneHotEncoder() ##\n",
    "\n",
    "Let's take a close look at this parameter.\n",
    "\n",
    "As per Spark [doc](https://spark.apache.org/docs/latest/api/python/pyspark.ml.html#pyspark.ml.feature.OneHotEncoder):\n",
    "\n",
    "\"A one-hot encoder that maps a column of category indices to a column of binary vectors, with at most a single one-value per row that indicates the input category index. For example with 5 categories, an input value of 2.0 would map to an output vector of [0.0, 0.0, 1.0, 0.0]. The last category is not included by default (configurable via dropLast) because it makes the vector entries sum up to one, and hence linearly dependent. So an input value of 4.0 maps to [0.0, 0.0, 0.0, 0.0].\"\n",
    "\n",
    "Linearly dependent? What does that mean? Red Huq has written a superb article on [this](https://inmachineswetrust.com/posts/drop-first-columns/).\n",
    "\n",
    "Comparing the two features column with or without `dropLast=False` via `df_final.collect()[0]['features']`:\n",
    "\n",
    "{with}  \n",
    "`SparseVector(61, {0: 6.0, 1: 1169.0, 2: 4.0, 3: 4.0, 4: 67.0, 5: 2.0, 6: 1.0, 8: 1.0, 12: 1.0, 16: 1.0, 27: 1.0, 32: 1.0, 36: 1.0, 40: 1.0, 44: 1.0, 47: 1.0, 50: 1.0, 53: 1.0, 58: 1.0, 59: 1.0})`\n",
    "\n",
    "{without}  \n",
    "`SparseVector(48, {0: 6.0, 1: 1169.0, 2: 4.0, 3: 4.0, 4: 67.0, 5: 2.0, 6: 1.0, 8: 1.0, 11: 1.0, 14: 1.0, 24: 1.0, 28: 1.0, 31: 1.0, 34: 1.0, 37: 1.0, 39: 1.0, 41: 1.0, 43: 1.0, 47: 1.0})`\n",
    "\n",
    "The difference between 61 and 48? 13! This represent 13 times one (1), an extra category from each column of the 13 that was one-hot-encoded, represented with the prefix \"_VEC\".\n",
    "\n",
    "Why does the {with} scenario has an element of **`12: 1.0`**, while the {without} does **`11: 1.0`**? Simply put, the first seven (7) elements i.e., from `0:` to `6:`, are identical because we pick up all the `smallint` datatypes first, no one-hot-encoded columns yet.\n",
    "\n",
    "The difference starts from that point onwards, when we take in the columns with the suffix of \"_VEC\".\n",
    "\n",
    "As for the \"1_VEC\", although both scenarios has one (1) flag in the second one-hot-encoded value column in the `8:`, the {with} scenario occupies between `7:` and `10:` whereas the {without} does between `7:` and `9:`, judging from `1_VEC=SparseVector(4, {1: 1.0})` vs. `1_VEC=SparseVector(3, {1: 1.0})`.\n",
    "\n",
    "(If the value is the first one-hot-encoded value, it will be `7: 1.0` and `1_VEC=SparseVector(4, {**0**: 1.0})` ({with}) and `1_VEC=SparseVector(3, {**0**: 1.0})` ({without}).)\n",
    "\n",
    "Next, the \"3_VEC\" one-hot-encoded columns occupy between `11:` and `15:` {with} and between `10:` and `13:` {without} respectively. Given `3_VEC=SparseVector(5, {1: 1.0})` vs. `3_VEC=SparseVector(4, {1: 1.0})`, the second one-hot-encoded value includes a value of 1, so **`12: 1.0`** ({with}) and **`11: 1.0`** ({without})."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing the columns with or without `dropLast=False` via `df.collect()[0]`:\n",
    "\n",
    "{with}  \n",
    "1='A11', 2=6, 3='A34', 4='A43', 5=1169, 6='A65', 7='A75', 8=4, 9='A93', 10='A101', 11=4, 12='A121', 13=67, 14='A143', 15='A152', 16=2, 17='A173', 18=1, 19='A192', 20='A201', target=1, 1_INDEX=1.0, 1_VEC=SparseVector(4, {1: 1.0}), 3_INDEX=1.0, 3_VEC=SparseVector(5, {1: 1.0}), 4_INDEX=0.0, 4_VEC=SparseVector(10, {0: 1.0}), 6_INDEX=1.0, 6_VEC=SparseVector(5, {1: 1.0}), 7_INDEX=1.0, 7_VEC=SparseVector(5, {1: 1.0}), 9_INDEX=0.0, 9_VEC=SparseVector(4, {0: 1.0}), 10_INDEX=0.0, 10_VEC=SparseVector(3, {0: 1.0}), 12_INDEX=1.0, 12_VEC=SparseVector(4, {1: 1.0}), 14_INDEX=0.0, 14_VEC=SparseVector(3, {0: 1.0}), 15_INDEX=0.0, 15_VEC=SparseVector(3, {0: 1.0}), 17_INDEX=0.0, 17_VEC=SparseVector(4, {0: 1.0}), 19_INDEX=1.0, **19_VEC=SparseVector(2, {1: 1.0})**, 20_INDEX=0.0, 20_VEC=SparseVector(2, {0: 1.0})\n",
    "\n",
    "{without}  \n",
    "1='A11', 2=6, 3='A34', 4='A43', 5=1169, 6='A65', 7='A75', 8=4, 9='A93', 10='A101', 11=4, 12='A121', 13=67, 14='A143', 15='A152', 16=2, 17='A173', 18=1, 19='A192', 20='A201', target=1, 1_INDEX=1.0, 1_VEC=SparseVector(3, {1: 1.0}), 3_INDEX=1.0, 3_VEC=SparseVector(4, {1: 1.0}), 4_INDEX=0.0, 4_VEC=SparseVector(9, {0: 1.0}), 6_INDEX=1.0, 6_VEC=SparseVector(4, {1: 1.0}), 7_INDEX=1.0, 7_VEC=SparseVector(4, {1: 1.0}), 9_INDEX=0.0, 9_VEC=SparseVector(3, {0: 1.0}), 10_INDEX=0.0, 10_VEC=SparseVector(2, {0: 1.0}), 12_INDEX=1.0, 12_VEC=SparseVector(3, {1: 1.0}), 14_INDEX=0.0, 14_VEC=SparseVector(2, {0: 1.0}), 15_INDEX=0.0, 15_VEC=SparseVector(2, {0: 1.0}), 17_INDEX=0.0, 17_VEC=SparseVector(3, {0: 1.0}), 19_INDEX=1.0, **19_VEC=SparseVector(1, {})**, 20_INDEX=0.0, 20_VEC=SparseVector(1, {0: 1.0})\n",
    "\n",
    "Take a note of the difference between the two 19_VEC columns with bianary outcomes.\n",
    "\n",
    "Also, check out Renjith Madhavan's [Sparse Vector vs. Dense Vector](https://youtu.be/oGwEv82ifrE)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA vs. SVD ##\n",
    "\n",
    "Andre Perunicic goes over SVD on his succinct [article](https://intoli.com/blog/pca-and-svd/).\n",
    "\n",
    "For someone like me who is penchant for LOTS of visuals, Cory Maklin has done [it](https://towardsdatascience.com/singular-value-decomposition-example-in-python-dab2507d85a0) - with numPy. \n",
    "\n",
    "[Nick walsh](https://stackoverflow.com/a/56012609) also attests that \"SVD is the more stable solution for preserving data integrity due to rounding inaccuracies as a result of computing the product of your dataset by its tranpose matrix (X*X⊤)\" as well as J.M.'s [Läuchli matrix example](https://math.stackexchange.com/a/359428) and Amoeba's [Eigenvalue Decomposition comparison](https://stats.stackexchange.com/a/87536).\n",
    "\n",
    "Bear in mind that SVD runs much SLOWER as it is computationally more INTENSIVE.\n",
    "\n",
    "Also, take note of Vaquar Khan's [mllib vs. ml](https://stackoverflow.com/a/58799652) and Mohamed Ali Jamaoui's [pySpark vs. Scipy](https://stackoverflow.com/a/58799652) comparions.\n",
    "\n",
    "In reference to [Elias Abou Haydar and Sergul Aydore's discussion](https://stackoverflow.com/a/38107324), let's import their SVD method as Java wrapper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.mllib.common import callMLlibFunc, JavaModelWrapper\n",
    "from pyspark.mllib.linalg.distributed import RowMatrix\n",
    "\n",
    "\n",
    "class SVD(JavaModelWrapper):\n",
    "    \"\"\"Wrapper around the SVD scala case class\"\"\"\n",
    "    @property\n",
    "    def U(self):\n",
    "        \"\"\" Returns a RowMatrix whose columns are the left singular vectors of the SVD if computeU was set to be True.\"\"\"\n",
    "        u = self.call(\"U\")\n",
    "        if u is not None:\n",
    "            return RowMatrix(u)\n",
    "\n",
    "    @property\n",
    "    def s(self):\n",
    "        \"\"\"Returns a DenseVector with singular values in descending order.\"\"\"\n",
    "        return self.call(\"s\")\n",
    "\n",
    "    @property\n",
    "    def V(self):\n",
    "        \"\"\" Returns a DenseMatrix whose columns are the right singular vectors of the SVD.\"\"\"\n",
    "        return self.call(\"V\")\n",
    "    \n",
    "def computeSVD(row_matrix, k, computeU=False, rCond=1e-9):\n",
    "    \"\"\"\n",
    "    Computes the singular value decomposition of the RowMatrix.\n",
    "    The given row matrix A of dimension (m X n) is decomposed into U * s * V'T where\n",
    "    * s: DenseVector consisting of square root of the eigenvalues (singular values) in descending order.\n",
    "    * U: (m X k) (left singular vectors) is a RowMatrix whose columns are the eigenvectors of (A X A')\n",
    "    * v: (n X k) (right singular vectors) is a Matrix whose columns are the eigenvectors of (A' X A)\n",
    "    :param k: number of singular values to keep. We might return less than k if there are numerically zero singular values.\n",
    "    :param computeU: Whether of not to compute U. If set to be True, then U is computed by A * V * sigma^-1\n",
    "    :param rCond: the reciprocal condition number. All singular values smaller than rCond * sigma(0) are treated as zero, where sigma(0) is the largest singular value.\n",
    "    :returns: SVD object\n",
    "    \"\"\"\n",
    "    java_model = row_matrix._java_matrix_wrapper.call(\"computeSVD\", int(k), computeU, float(rCond))\n",
    "    return SVD(java_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Work in Progress - Obtain SVD ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pyspark.ml.feature.PCA'>\n"
     ]
    }
   ],
   "source": [
    "#from pyspark.mllib.linalg import Vectors\n",
    "#from pyspark.sql import sqlContext\n",
    "#from pyspark.mllib.utils import MLUtils\n",
    "#from pyspark.mllib.util import MLUtils\n",
    "\n",
    "\n",
    "RAW_FEATURES = \"raw_feasures\"\n",
    "PCA_FEATURES = \"pca_features\"\n",
    "PCA_FEATURES_NUM = 10\n",
    "\n",
    "pca_extracted = PCA(k=PCA_FEATURES_NUM, inputCol=RAW_FEATURES, outputCol=PCA_FEATURES)\n",
    "# type(pca_extracted) # pyspark.ml.feature.PCA\n",
    "\n",
    "model = pca_extracted.fit(df_pca)\n",
    "# type(model) # pyspark.ml.feature.PCAModel\n",
    "\n",
    "features = model.transform(df_pca) # this create a DataFrame with the regular features and pca_features\n",
    "# type(features) # pyspark.sql.dataframe.DataFrame\n",
    "features.collect()[0][PCA_FEATURES]\n",
    "\n",
    "# # We can now extract the pca_features to prepare our RowMatrix.\n",
    "pca_features = features.select(PCA_FEATURES).rdd.map(lambda row : row[0])\n",
    "# type(pca_features) # pyspark.rdd.PipelinedRDD\n",
    "\n",
    "\n",
    "# mat = RowMatrix(pca_features)\n",
    "# type(mat) # pyspark.mllib.linalg.distributed.RowMatrix\n",
    "\n",
    "#svd_model = RowMatrix(pca_features).computeSVD(PCA_FEATURES_NUM, True)\n",
    "#svd_model = mat.computeSVD(PCA_FEATURES_NUM, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Input dataset must be a DataFrame but got <class 'pyspark.rdd.PipelinedRDD'>.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-87ed7404813e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;31m# type(df_pca.collect()[0][RAW_FEATURES])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mpca_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMLUtils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvertVectorColumnsToML\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpca_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPCA_FEATURES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpca_features\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mPCA_FEATURES\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/arcgis/lib/python3.7/site-packages/pyspark/mllib/util.py\u001b[0m in \u001b[0;36mconvertVectorColumnsToML\u001b[0;34m(dataset, *cols)\u001b[0m\n\u001b[1;32m    239\u001b[0m         \"\"\"\n\u001b[1;32m    240\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 241\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Input dataset must be a DataFrame but got {}.\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    242\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcallMLlibFunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"convertVectorColumnsToML\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Input dataset must be a DataFrame but got <class 'pyspark.rdd.PipelinedRDD'>."
     ]
    }
   ],
   "source": [
    "# TypeError: Cannot convert type <class 'pyspark.ml.linalg.DenseVector'> into Vector\n",
    "#svd = computeSVD(mat,PCA_FEATURES_NUM,True)\n",
    "#svd = mat.computeSVD(PCA_FEATURES_NUM,True)\n",
    "\n",
    "# from pyspark.mllib.utils import MLUtils\n",
    "from pyspark.mllib.util import MLUtils\n",
    "# df_pca = MLUtils.convertVectorColumnsFromML(df_pca, RAW_FEATURES)\n",
    "#df_pca = MLUtils.convertVectorColumnsToML(df_pca, RAW_FEATURES)\n",
    "# df_pca.collect()[0][RAW_FEATURES]\n",
    "# type(df_pca.collect()[0][RAW_FEATURES])\n",
    "\n",
    "pca_features = MLUtils.convertVectorColumnsToML(pca_features, PCA_FEATURES)\n",
    "type(pca_features.collect()[0][PCA_FEATURES])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}