Apply yapf (#160)

* make comlumn limit large

* apply yapf all

examples/sample_pandas_type_check.py

@@ -24,7 +24,4 @@ def run(self):
 
 
 if __name__ == '__main__':
-    gokart.run([
-        'sample_pandas_type_check.SampleTask',
-        '--local-scheduler',
-        '--rerun'])
+    gokart.run(['sample_pandas_type_check.SampleTask', '--local-scheduler', '--rerun'])

gokart/file_processor.py

@@ -13,7 +13,6 @@
 
 from gokart.object_storage import ObjectStorage
 
-
 logger = getLogger(__name__)
 
 

gokart/gcs_config.py

@@ -13,17 +13,11 @@
 
 
 class GCSConfig(luigi.Config):
-    gcs_credential_name: str = luigi.Parameter(
-        default='GCS_CREDENTIAL', description='GCS credential environment variable.')
-    discover_cache_local_path: str = luigi.Parameter(
-        default='DISCOVER_CACHE_LOCAL_PATH', description='The file path of discover api cache.')
+    gcs_credential_name: str = luigi.Parameter(default='GCS_CREDENTIAL', description='GCS credential environment variable.')
+    discover_cache_local_path: str = luigi.Parameter(default='DISCOVER_CACHE_LOCAL_PATH', description='The file path of discover api cache.')
 
-    _DISCOVERY_URI = (
-        "https://www.googleapis.com/discovery/v1/apis/" "{api}/{apiVersion}/rest"
-    )
-    _V2_DISCOVERY_URI = (
-        "https://{api}.googleapis.com/$discovery/rest?" "version={apiVersion}"
-    )
+    _DISCOVERY_URI = ("https://www.googleapis.com/discovery/v1/apis/" "{api}/{apiVersion}/rest")
+    _V2_DISCOVERY_URI = ("https://{api}.googleapis.com/$discovery/rest?" "version={apiVersion}")
     _client = None
 
     def get_gcs_client(self) -> luigi.contrib.gcs.GCSClient:
@@ -42,9 +36,7 @@ def _get_gcs_client(self) -> luigi.contrib.gcs.GCSClient:
                     for discovery_url in (self._DISCOVERY_URI, self._V2_DISCOVERY_URI):
                         requested_url = uritemplate.expand(discovery_url, params)
                         try:
-                            content = _retrieve_discovery_doc(
-                                requested_url, discovery_http, False
-                            )
+                            content = _retrieve_discovery_doc(requested_url, discovery_http, False)
                         except HttpError as e:
                             if e.resp.status == http_client.NOT_FOUND:
                                 continue

gokart/gcs_zip_client.py

@@ -37,4 +37,3 @@ def _temporary_file_path(self):
         if base_name.endswith('/'):
             base_name = base_name[:-1]
         return base_name + extension
-

gokart/pandas_type_config.py

@@ -37,13 +37,14 @@ def _check_column(cls, df, column_name, column_type):
 
 class PandasTypeConfigMap(luigi.Config):
     """To initialize this class only once, this inherits luigi.Config."""
-
     def __init__(self, *args, **kwargs) -> None:
         super(PandasTypeConfigMap, self).__init__(*args, **kwargs)
         task_names = Register.task_names()
         task_classes = [Register.get_task_cls(task_name) for task_name in task_names]
-        self._map = {task_class.task_namespace: task_class for task_class in task_classes if
-                     issubclass(task_class, PandasTypeConfig) and task_class != PandasTypeConfig}
+        self._map = {
+            task_class.task_namespace: task_class
+            for task_class in task_classes if issubclass(task_class, PandasTypeConfig) and task_class != PandasTypeConfig
+        }
 
     def check(self, obj, task_namespace: str):
         if type(obj) == pd.DataFrame and task_namespace in self._map:

gokart/run.py

@@ -81,8 +81,7 @@ def _try_get_slack_api(cmdline_args: List[str]) -> Optional[gokart.slack.SlackAP
     return None
 
 
-def _try_to_send_event_summary_to_slack(slack_api: Optional[gokart.slack.SlackAPI],
-                                        event_aggregator: gokart.slack.EventAggregator, cmdline_args: List[str]):
+def _try_to_send_event_summary_to_slack(slack_api: Optional[gokart.slack.SlackAPI], event_aggregator: gokart.slack.EventAggregator, cmdline_args: List[str]):
     if slack_api is None:
         # do nothing
         return
@@ -93,13 +92,7 @@ def _try_to_send_event_summary_to_slack(slack_api: Optional[gokart.slack.SlackAP
         task_name = type(task).__name__
 
     comment = f'Report of {task_name}' + os.linesep + event_aggregator.get_summary()
-    content = os.linesep.join([
-        '===== Event List ====',
-        event_aggregator.get_event_list(),
-        os.linesep,
-        '==== Tree Info ====',
-        tree_info
-    ])
+    content = os.linesep.join(['===== Event List ====', event_aggregator.get_event_list(), os.linesep, '==== Tree Info ====', tree_info])
     slack_api.send_snippet(comment=comment, title='event.txt', content=content)
 
 

gokart/s3_config.py

@@ -4,12 +4,9 @@
 
 
 class S3Config(luigi.Config):
-    aws_access_key_id_name = luigi.Parameter(
-        default='AWS_ACCESS_KEY_ID', description='AWS access key id environment variable.')
-    aws_secret_access_key_name = luigi.Parameter(
-        default='AWS_SECRET_ACCESS_KEY', description='AWS secret access key environment variable.')
+    aws_access_key_id_name = luigi.Parameter(default='AWS_ACCESS_KEY_ID', description='AWS access key id environment variable.')
+    aws_secret_access_key_name = luigi.Parameter(default='AWS_SECRET_ACCESS_KEY', description='AWS secret access key environment variable.')
 
     def get_s3_client(self) -> luigi.contrib.s3.S3Client:
-        return luigi.contrib.s3.S3Client(
-            aws_access_key_id=os.environ.get(self.aws_access_key_id_name),
-            aws_secret_access_key=os.environ.get(self.aws_secret_access_key_name))
+        return luigi.contrib.s3.S3Client(aws_access_key_id=os.environ.get(self.aws_access_key_id_name),
+                                         aws_secret_access_key=os.environ.get(self.aws_secret_access_key_name))

gokart/slack/event_aggregator.py

@@ -22,10 +22,8 @@ def get_summary(self) -> str:
     def get_event_list(self) -> str:
         message = ''
         if self._event_queue[luigi.Event.FAILURE]:
-            failure_message = os.linesep.join([
-                f"Task: {failure['task']}; Exception: {failure['exception']}"
-                for failure in self._event_queue[luigi.Event.FAILURE]
-            ])
+            failure_message = os.linesep.join(
+                [f"Task: {failure['task']}; Exception: {failure['exception']}" for failure in self._event_queue[luigi.Event.FAILURE]])
             message += '---- Failure Tasks ----' + os.linesep + failure_message
         if self._event_queue[luigi.Event.SUCCESS]:
             success_message = os.linesep.join(self._event_queue[luigi.Event.SUCCESS])

gokart/slack/slack_config.py

@@ -5,6 +5,7 @@ class SlackConfig(luigi.Config):
     token_name = luigi.Parameter(default='SLACK_TOKEN', description='slack token environment variable.')
     channel = luigi.Parameter(default='', significant=False, description='channel name for notification.')
     to_user = luigi.Parameter(default='', significant=False, description='Optional; user name who is supposed to be mentioned.')
-    send_tree_info = luigi.BoolParameter(default=False, significant=False,
+    send_tree_info = luigi.BoolParameter(default=False,
+                                         significant=False,
                                          description='When this option is true, the dependency tree of tasks is included in send message.'
-                                                     'It is recommended to set false to this option when notification takes long time.')
+                                         'It is recommended to set false to this option when notification takes long time.')

gokart/target.py

@@ -74,10 +74,10 @@ def _path(self) -> str:
 
 class SingleFileTarget(TargetOnKart):
     def __init__(
-            self,
-            target: luigi.target.FileSystemTarget,
-            processor: FileProcessor,
-            redis_params: RedisParams,
+        self,
+        target: luigi.target.FileSystemTarget,
+        processor: FileProcessor,
+        redis_params: RedisParams,
     ) -> None:
         self._target = target
         self._processor = processor
@@ -110,12 +110,12 @@ def _path(self) -> str:
 
 class ModelTarget(TargetOnKart):
     def __init__(
-            self,
-            file_path: str,
-            temporary_directory: str,
-            load_function,
-            save_function,
-            redis_params: RedisParams,
+        self,
+        file_path: str,
+        temporary_directory: str,
+        load_function,
+        save_function,
+        redis_params: RedisParams,
     ) -> None:
         self._zip_client = make_zip_client(file_path, temporary_directory)
         self._temporary_directory = temporary_directory

gokart/zip_client_util.py

@@ -1,6 +1,7 @@
 from gokart.zip_client import ZipClient, LocalZipClient
 from gokart.object_storage import ObjectStorage
 
+
 def make_zip_client(file_path: str, temporary_directory: str) -> ZipClient:
     if ObjectStorage.if_object_storage_path(file_path):
         return ObjectStorage.get_zip_client(file_path=file_path, temporary_directory=temporary_directory)

setup.py

@@ -20,11 +20,7 @@
     'redis',
 ]
 
-tests_require = [
-    'moto==1.3.6',
-    'testfixtures==6.14.2',
-    'matplotlib'
-]
+tests_require = ['moto==1.3.6', 'testfixtures==6.14.2', 'matplotlib']
 
 setup(
     name='gokart',

test/slack/test_slack_api.py

@@ -34,8 +34,7 @@ def _conversations_list(params={}):
 
         with LogCapture() as l:
             gokart.slack.SlackAPI(token='invalid', channel='test', to_user='test user')
-            l.check(('gokart.slack.slack_api', 'WARNING',
-                     'The job will start without slack notification: Channel test is not found in public channels.'))
+            l.check(('gokart.slack.slack_api', 'WARNING', 'The job will start without slack notification: Channel test is not found in public channels.'))
 
     @mock.patch('gokart.slack.slack_api.slack_sdk.WebClient')
     def test_invalid_channel(self, patch):
@@ -58,10 +57,8 @@ def _conversations_list(params={}):
 
         with LogCapture() as l:
             gokart.slack.SlackAPI(token='valid', channel='invalid_channel', to_user='test user')
-            l.check((
-                'gokart.slack.slack_api', 'WARNING',
-                'The job will start without slack notification: Channel invalid_channel is not found in public channels.'
-            ))
+            l.check(('gokart.slack.slack_api', 'WARNING',
+                     'The job will start without slack notification: Channel invalid_channel is not found in public channels.'))
 
     @mock.patch('gokart.slack.slack_api.slack_sdk.WebClient')
     def test_send_snippet_with_invalid_token(self, patch):
@@ -90,9 +87,7 @@ def _api_call(method, data={}):
         with LogCapture() as l:
             api = gokart.slack.SlackAPI(token='valid', channel='valid', to_user='test user')
             api.send_snippet(comment='test', title='title', content='content')
-            l.check(
-                ('gokart.slack.slack_api', 'WARNING',
-                 'Failed to send slack notification: Error while uploading file. The error reason is "error_reason".'))
+            l.check(('gokart.slack.slack_api', 'WARNING', 'Failed to send slack notification: Error while uploading file. The error reason is "error_reason".'))
 
     @mock.patch('gokart.slack.slack_api.slack_sdk.WebClient')
     def test_send(self, patch):

test/test_pandas_type_config.py

@@ -10,7 +10,6 @@
 
 
 class _DummyPandasTypeConfig(PandasTypeConfig):
-
     @classmethod
     def type_dict(cls) -> Dict[str, Any]:
         return {'int_column': int, 'datetime_column': datetime, 'array_column': np.ndarray}

test/test_run.py

@@ -5,7 +5,6 @@
 import luigi
 import luigi.mock
 
-
 import gokart
 from gokart.slack import SlackConfig
 from gokart.run import _try_to_send_event_summary_to_slack
@@ -38,7 +37,11 @@ def test_run_with_undefined_environ(self):
         with self.assertRaises(luigi.parameter.MissingParameterException) as missing_parameter:
             gokart.run()
 
-    @patch('sys.argv', new=['main', '--tree-info-mode=simple', '--tree-info-output-path=tree.txt', f'{__name__}._DummyTask', '--param', 'test', '--log-level=CRITICAL', '--local-scheduler'])
+    @patch('sys.argv',
+           new=[
+               'main', '--tree-info-mode=simple', '--tree-info-output-path=tree.txt', f'{__name__}._DummyTask', '--param', 'test', '--log-level=CRITICAL',
+               '--local-scheduler'
+           ])
     @patch('luigi.LocalTarget', new=lambda path, **kwargs: luigi.mock.MockTarget(path, **kwargs))
     def test_run_tree_info(self):
         config_file_path = os.path.join(os.path.dirname(__name__), 'config', 'test_config.ini')
@@ -58,6 +61,7 @@ def test_try_to_send_event_summary_to_slack(self, make_tree_info_mock: MagicMock
 
         def get_content(content: str, **kwargs):
             self.output = content
+
         slack_api_mock = MagicMock()
         slack_api_mock.send_snippet.side_effect = get_content
 
@@ -74,10 +78,8 @@ def get_content(content: str, **kwargs):
             _try_to_send_event_summary_to_slack(slack_api_mock, event_aggregator_mock, cmdline_args)
         expects = os.linesep.join([
             '===== Event List ====',
-            event_aggregator_mock.get_event_list(),
-            os.linesep,
-            '==== Tree Info ====',
-            'Please add SlackConfig.send_tree_info to include tree-info'])
+            event_aggregator_mock.get_event_list(), os.linesep, '==== Tree Info ====', 'Please add SlackConfig.send_tree_info to include tree-info'
+        ])
 
         results = self.output
         self.assertEqual(expects, results)

test/testing/test_run_with_empty_data_frame.py

@@ -85,8 +85,10 @@ def test_run_with_error(self):
         self.assertTrue('DummyModelTask' in log_str)
 
     def test_run_with_namespace(self):
-        argv = [f'{__name__}.DummyWorkFlowWithoutError', '--local-scheduler', '--test-run-pandas', f'--test-run-namespace={__name__}', '--log-level=CRITICAL',
-                '--no-lock']
+        argv = [
+            f'{__name__}.DummyWorkFlowWithoutError', '--local-scheduler', '--test-run-pandas', f'--test-run-namespace={__name__}', '--log-level=CRITICAL',
+            '--no-lock'
+        ]
         logger = logging.getLogger('gokart.testing.check_if_run_with_empty_data_frame')
         with patch.object(logger, 'info') as mock_debug:
             with self.assertRaises(SystemExit) as exit_code:

yapf.ini

@@ -1,3 +1,3 @@
 [style]
 based_on_style = pep8
-column_limit = 120
+column_limit = 160