diff --git a/.github/workflows/monthly_collection.yaml b/.github/workflows/monthly_collection.yaml index 72a5693..3c18314 100644 --- a/.github/workflows/monthly_collection.yaml +++ b/.github/workflows/monthly_collection.yaml @@ -8,7 +8,7 @@ on: jobs: monthly_github_collection: - environment: monthly + environment: weekly runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -36,7 +36,7 @@ jobs: - name: Consolidate GitHub Data run: | uv run gitmetrics consolidate \ - --config-file monthly_extraction_config.yaml \ + --config-file weekly_extraction_config.yaml monthly_extraction_config.yaml \ --output-folder ${{ secrets.OUTPUT_FOLDER }} env: PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }} diff --git a/.github/workflows/weekly_collection.yaml b/.github/workflows/weekly_collection.yaml index 3b71bec..a8b3ac7 100644 --- a/.github/workflows/weekly_collection.yaml +++ b/.github/workflows/weekly_collection.yaml @@ -32,7 +32,7 @@ jobs: - name: Consolidate GitHub Data run: | uv run gitmetrics consolidate \ - --config-file weekly_extraction_config.yaml \ + --config-file weekly_extraction_config.yaml monthly_extraction_config.yaml \ --output-folder ${{ secrets.OUTPUT_FOLDER }} env: PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }} diff --git a/gitmetrics/__main__.py b/gitmetrics/__main__.py index bd073a2..72c6f39 100644 --- a/gitmetrics/__main__.py +++ b/gitmetrics/__main__.py @@ -140,11 +140,13 @@ def _summarize(args, parser): def _consolidate(args, parser): - config = _load_config(args.config_file) - projects = config['projects'] + all_projects = [] + for config_file in args.config_file: + config = _load_config(config_file) + all_projects.extend(config.get('projects', [])) consolidate_metrics( - projects=projects, + projects=all_projects, output_folder=args.output_folder, dry_run=args.dry_run, verbose=args.verbose, @@ -223,8 +225,9 @@ def _get_parser(): '-c', '--config-file', type=str, - default=DEFAULT_PROJECT_DEFINITIONS, - help='Path to the configuration file.', + nargs='+', + default=[DEFAULT_PROJECT_DEFINITIONS], + help='Path(s) to configuration file(s).', ) consolidate.add_argument( '-d', diff --git a/gitmetrics/consolidate.py b/gitmetrics/consolidate.py index f0998ff..666999d 100644 --- a/gitmetrics/consolidate.py +++ b/gitmetrics/consolidate.py @@ -12,6 +12,7 @@ METRICS_SHEET_NAME, VALUE_COLUMN_NAME, ) +from gitmetrics.drive import _get_drive_client, is_drive_path, split_drive_path from gitmetrics.output import create_spreadsheet, load_spreadsheet OUTPUT_FILENAME = 'gitmetrics_consolidated_summary_to_date' @@ -55,6 +56,7 @@ def consolidate_metrics(projects, output_folder, dry_run=False, verbose=True): row_info.update(row_values) if verbose: LOGGER.info(f' {project} values: {row_info}') + rows.append(row_info) consolidated_df = pd.DataFrame(rows) @@ -62,6 +64,23 @@ def consolidate_metrics(projects, output_folder, dry_run=False, verbose=True): if verbose: LOGGER.info(f'Sheet Name: {SHEET_NAME}') LOGGER.info(consolidated_df.to_string()) + if not dry_run: output_path = os.path.join(output_folder, OUTPUT_FILENAME) + + if is_drive_path(output_folder): + drive = _get_drive_client() + gdrive_folder = output_folder.rstrip('/') + '/' + folder_id, _ = split_drive_path(gdrive_folder) + + folder = drive.CreateFile({'id': folder_id}) + folder.FetchMetadata(fields='parents') + + parents = folder.get('parents') or [] + parent_id = parents[0].get('id') + + output_path = f'gdrive://{parent_id}/{OUTPUT_FILENAME}' + create_spreadsheet(output_path=output_path, sheets=sheets) + + else: create_spreadsheet(output_path=output_path, sheets=sheets) diff --git a/gitmetrics/drive.py b/gitmetrics/drive.py index 210070f..5094936 100644 --- a/gitmetrics/drive.py +++ b/gitmetrics/drive.py @@ -98,7 +98,7 @@ def upload_spreadsheet(content, filename, folder): retries = 0 while retries != MAX_UPLOAD_RETRIES: try: - if drive_file['mimeType'] == SPREADSHEET_MIMETYPE: + if drive_file.get('mimeType') == SPREADSHEET_MIMETYPE: drive_file.Upload() else: drive_file.Upload({'convert': True}) diff --git a/monthly_extraction_config.yaml b/monthly_extraction_config.yaml index 31c3f75..47d6803 100644 --- a/monthly_extraction_config.yaml +++ b/monthly_extraction_config.yaml @@ -8,4 +8,3 @@ projects: scikit-learn: ray: airbyte: - PyTorchLightning: diff --git a/weekly_extraction_config.yaml b/weekly_extraction_config.yaml index e3fb698..077a22d 100644 --- a/weekly_extraction_config.yaml +++ b/weekly_extraction_config.yaml @@ -11,3 +11,4 @@ projects: pycaret: snorkel: spacy: + PyTorchLightning: