From 6343ee6db399a2d1f92e1cbbde2a82c8b60f3eeb Mon Sep 17 00:00:00 2001 From: Neill Cox Date: Mon, 23 Oct 2023 20:36:23 +1100 Subject: [PATCH 1/5] :construction: WIP: Begin exploring the GitLab API - add PrettyTable for display This commit is very much WIP. Just some inital playing with the API. Eventually this should lead to a tool that will export at a minimum the wiki and issues for a project. First though, I need to get a feel for how the API works. --- pyproject.toml | 3 +- src/gitea_gitlab_exporter/__init__.py | 161 +++++++++++++++++++++++++- 2 files changed, 158 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 221b3fa..069d0e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,8 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "requests" + "prettytable", + "requests", ] [project.urls] diff --git a/src/gitea_gitlab_exporter/__init__.py b/src/gitea_gitlab_exporter/__init__.py index b5a8af4..e1e27b3 100644 --- a/src/gitea_gitlab_exporter/__init__.py +++ b/src/gitea_gitlab_exporter/__init__.py @@ -6,13 +6,164 @@ Copyright 2023 Neill Cox """ +import argparse +import json import logging +import os +import sys + +import requests +from prettytable import PrettyTable -def hello_world(): - logging.debug("hello_world called") - print("Hello World!") +def gitlab_url(path): + return "https://gitlab.com/api/v4" + path -if __name__ == "__main__": - hello_world() +def get(url, args): + logging.debug("get called") + logging.debug("url: %s", url) + + response = requests.get(url, headers={"PRIVATE-TOKEN": args.gitlab_token}) + + logging.debug("response.status: %s", response.status_code) + logging.debug("body: %s", response.text) + + return response.json() + + +def parse_args(): + """Parse the command line arguments""" + + parser = argparse.ArgumentParser() + parser.add_argument( + "-t", + "--gitlab-token", + help=( + "A private access token to access GitLab with. If not specified " + "will use $GL_TOKEN. Required." + ), + ) + parser.add_argument("--debug", action="store_true") + parser.add_argument("--log-file", default="gitlab2gitea.log") + + subparsers = parser.add_subparsers() + + lp_sp = subparsers.add_parser("list-projects") + lp_sp.set_defaults(func=list_projects) + + lp_sp = subparsers.add_parser("get-user") + lp_sp.set_defaults(func=cli_get_user) + + lp_sp = subparsers.add_parser("get-project-details") + lp_sp.add_argument("--project", required=True) + lp_sp.set_defaults(func=get_project_details) + + args = parser.parse_args() + + log_level = logging.INFO + if args.debug: + log_level = logging.DEBUG + + logging.basicConfig(filename=args.log_file, level=log_level) + + if args.gitlab_token is None: + args.gitlab_token = os.environ.get("GL_TOKEN") + + if args.gitlab_token is None: + err_str = "/".join( + [ac for ac in parser._actions if ac.dest == "gitlab_token"][ + 0 + ].option_strings + ) + parser.print_usage() + + print( + f"{parser.prog}: error: the following arguments are " + f"required: {err_str}" + ) + sys.exit(2) + + if "func" in args: + args.func(args) + + return args + + +def cli_get_user(args): + logging.debug("cli_get_user called") + + user = get_user(args) + + print(json.dumps(user)) + + +def get_user(args): + logging.debug("get_user called") + + url = gitlab_url("/user") + + result = get(url, args) + + return result + + +def list_projects(args): + logging.debug("list_projects called") + user_id = get_user(args)["id"] + + url = gitlab_url( + f"/users/{user_id}/projects?pagination=offset&per_page=500&" + "order_by=name&sort=asc" + ) + + result = get(url, args) + + tbl = PrettyTable() + tbl.align = "l" + tbl.field_names = [ + "ID", + "Name", + # "Description" + ] + + for row in result: + tbl.add_row( + [ + row["id"], + row["name"], + # row["description"][:50] if row["description"] else "" + ] + ) + + print(tbl) + + +def get_project_details(args): + user_id = get_user(args)["id"] + url = gitlab_url( + f"/users/{user_id}/projects?pagination=offset&per_page=500&" + "order_by=name&sort=asc" + ) + + response = get(url, args) + + project = [ + project + for project in response + if (project["name"] == args.project or project["id"] == args.project) + ][0] + + if not project: + raise KeyError(f"Project {args.project} not found") + + print(json.dumps(project)) + + +def exporter(): + args = parse_args() + list_projects(args) + + # user = get_user(args) + # args.user_id = user['id'] + # print(json.dumps(get_project_details(args))) From 024f01c9d13764afe2b70f947e0a8f46172639ab Mon Sep 17 00:00:00 2001 From: Neill Cox Date: Tue, 24 Oct 2023 11:25:26 +1100 Subject: [PATCH 2/5] :construction: reorganise code --- src/gitea_gitlab_exporter/__init__.py | 161 +------------------------- src/gitea_gitlab_exporter/api.py | 49 ++++++++ src/gitea_gitlab_exporter/cli.py | 106 +++++++++++++++++ src/gitea_gitlab_exporter/utils.py | 19 +++ 4 files changed, 177 insertions(+), 158 deletions(-) create mode 100644 src/gitea_gitlab_exporter/api.py create mode 100644 src/gitea_gitlab_exporter/cli.py create mode 100644 src/gitea_gitlab_exporter/utils.py diff --git a/src/gitea_gitlab_exporter/__init__.py b/src/gitea_gitlab_exporter/__init__.py index e1e27b3..fa7e287 100644 --- a/src/gitea_gitlab_exporter/__init__.py +++ b/src/gitea_gitlab_exporter/__init__.py @@ -1,169 +1,14 @@ """ -A skeleton for a python project. +A tool to copy a project from gitlab to gitea Copyright 2023 Neill Cox """ -import argparse -import json -import logging -import os -import sys - -import requests -from prettytable import PrettyTable - - -def gitlab_url(path): - return "https://gitlab.com/api/v4" + path - - -def get(url, args): - logging.debug("get called") - logging.debug("url: %s", url) - - response = requests.get(url, headers={"PRIVATE-TOKEN": args.gitlab_token}) - - logging.debug("response.status: %s", response.status_code) - logging.debug("body: %s", response.text) - - return response.json() - - -def parse_args(): - """Parse the command line arguments""" - - parser = argparse.ArgumentParser() - parser.add_argument( - "-t", - "--gitlab-token", - help=( - "A private access token to access GitLab with. If not specified " - "will use $GL_TOKEN. Required." - ), - ) - parser.add_argument("--debug", action="store_true") - parser.add_argument("--log-file", default="gitlab2gitea.log") - - subparsers = parser.add_subparsers() - - lp_sp = subparsers.add_parser("list-projects") - lp_sp.set_defaults(func=list_projects) - - lp_sp = subparsers.add_parser("get-user") - lp_sp.set_defaults(func=cli_get_user) - - lp_sp = subparsers.add_parser("get-project-details") - lp_sp.add_argument("--project", required=True) - lp_sp.set_defaults(func=get_project_details) - - args = parser.parse_args() - - log_level = logging.INFO - if args.debug: - log_level = logging.DEBUG - - logging.basicConfig(filename=args.log_file, level=log_level) - - if args.gitlab_token is None: - args.gitlab_token = os.environ.get("GL_TOKEN") - - if args.gitlab_token is None: - err_str = "/".join( - [ac for ac in parser._actions if ac.dest == "gitlab_token"][ - 0 - ].option_strings - ) - parser.print_usage() - - print( - f"{parser.prog}: error: the following arguments are " - f"required: {err_str}" - ) - sys.exit(2) - - if "func" in args: - args.func(args) - - return args - - -def cli_get_user(args): - logging.debug("cli_get_user called") - - user = get_user(args) - - print(json.dumps(user)) - - -def get_user(args): - logging.debug("get_user called") - - url = gitlab_url("/user") - - result = get(url, args) - - return result - - -def list_projects(args): - logging.debug("list_projects called") - user_id = get_user(args)["id"] - - url = gitlab_url( - f"/users/{user_id}/projects?pagination=offset&per_page=500&" - "order_by=name&sort=asc" - ) - - result = get(url, args) - - tbl = PrettyTable() - tbl.align = "l" - tbl.field_names = [ - "ID", - "Name", - # "Description" - ] - - for row in result: - tbl.add_row( - [ - row["id"], - row["name"], - # row["description"][:50] if row["description"] else "" - ] - ) - - print(tbl) - - -def get_project_details(args): - user_id = get_user(args)["id"] - url = gitlab_url( - f"/users/{user_id}/projects?pagination=offset&per_page=500&" - "order_by=name&sort=asc" - ) - - response = get(url, args) - - project = [ - project - for project in response - if (project["name"] == args.project or project["id"] == args.project) - ][0] - - if not project: - raise KeyError(f"Project {args.project} not found") - - print(json.dumps(project)) +from .cli import cli_list_projects, parse_args def exporter(): args = parse_args() - list_projects(args) - - # user = get_user(args) - # args.user_id = user['id'] - # print(json.dumps(get_project_details(args))) + cli_list_projects(args) diff --git a/src/gitea_gitlab_exporter/api.py b/src/gitea_gitlab_exporter/api.py new file mode 100644 index 0000000..ef7e7d4 --- /dev/null +++ b/src/gitea_gitlab_exporter/api.py @@ -0,0 +1,49 @@ +import json +import logging + +from .utils import get, gitlab_url + + +def get_user(args): + logging.debug("get_user called") + + url = gitlab_url("/user") + + result = get(url, args) + + return result + + +def list_projects(args): + logging.debug("list_projects called") + user_id = get_user(args)["id"] + + url = gitlab_url( + f"/users/{user_id}/projects?pagination=offset&per_page=500&" + "order_by=name&sort=asc" + ) + + result = get(url, args) + + return result + + +def get_project_details(args): + user_id = get_user(args)["id"] + url = gitlab_url( + f"/users/{user_id}/projects?pagination=offset&per_page=500&" + "order_by=name&sort=asc" + ) + + response = get(url, args) + + project = [ + project + for project in response + if (project["name"] == args.project or project["id"] == args.project) + ][0] + + if not project: + raise KeyError(f"Project {args.project} not found") + + print(json.dumps(project)) diff --git a/src/gitea_gitlab_exporter/cli.py b/src/gitea_gitlab_exporter/cli.py new file mode 100644 index 0000000..0c60986 --- /dev/null +++ b/src/gitea_gitlab_exporter/cli.py @@ -0,0 +1,106 @@ +import argparse +import json +import logging +import os +import sys + +from prettytable import PrettyTable + +from .api import get_project_details, get_user, list_projects + + +def parse_args(): + """Parse the command line arguments""" + + parser = argparse.ArgumentParser() + parser.add_argument( + "-t", + "--gitlab-token", + help=( + "A private access token to access GitLab with. If not specified " + "will use $GL_TOKEN. Required." + ), + ) + parser.add_argument("--debug", action="store_true") + parser.add_argument("--log-file", default="gitlab2gitea.log") + parser.add_argument("-f", "--format", default="json") + + subparsers = parser.add_subparsers() + + lp_sp = subparsers.add_parser("list-projects") + lp_sp.set_defaults(func=list_projects) + + lp_sp = subparsers.add_parser("get-user") + lp_sp.set_defaults(func=cli_get_user) + + lp_sp = subparsers.add_parser("get-project-details") + lp_sp.add_argument("--project", required=True) + lp_sp.set_defaults(func=get_project_details) + + args = parser.parse_args() + + log_level = logging.INFO + if args.debug: + log_level = logging.DEBUG + + logging.basicConfig(filename=args.log_file, level=log_level) + + if args.gitlab_token is None: + args.gitlab_token = os.environ.get("GL_TOKEN") + + if args.gitlab_token is None: + err_str = "/".join( + [ac for ac in parser._actions if ac.dest == "gitlab_token"][ + 0 + ].option_strings + ) + parser.print_usage() + + print( + f"{parser.prog}: error: the following arguments are " + f"required: {err_str}" + ) + sys.exit(2) + + if "func" in args: + args.func(args) + + return args + + +def cli_get_user(args): + logging.debug("cli_get_user called") + + user = get_user(args) + + print(json.dumps(user)) + + +def cli_list_projects(args): + logging.debug("cli_list_projects called") + + projects = list_projects(args) + + if args.format == "table": + tbl = PrettyTable() + tbl.align = "l" + tbl.field_names = [ + "ID", + "Name", + # "Description" + ] + + for row in projects: + tbl.add_row( + [ + row["id"], + row["name"], + # row["description"][:50] if row["description"] else "" + ] + ) + + print(tbl) + elif args.format == "json": + print(json.dumps(projects)) + else: + print(projects) diff --git a/src/gitea_gitlab_exporter/utils.py b/src/gitea_gitlab_exporter/utils.py new file mode 100644 index 0000000..45a54d4 --- /dev/null +++ b/src/gitea_gitlab_exporter/utils.py @@ -0,0 +1,19 @@ +import logging + +import requests + + +def gitlab_url(path): + return "https://gitlab.com/api/v4" + path + + +def get(url, args): + logging.debug("get called") + logging.debug("url: %s", url) + + response = requests.get(url, headers={"PRIVATE-TOKEN": args.gitlab_token}) + + logging.debug("response.status: %s", response.status_code) + logging.debug("body: %s", response.text) + + return response.json() From 02f4919da2e3b17f48eeab36d5e67437900bccef Mon Sep 17 00:00:00 2001 From: Neill Cox Date: Tue, 24 Oct 2023 13:10:59 +1100 Subject: [PATCH 3/5] :construction: change to use click instead of argparse --- pyproject.toml | 3 +- src/gitea_gitlab_exporter/__init__.py | 7 -- src/gitea_gitlab_exporter/cli.py | 101 ++++++++++++++++++++++++-- src/gitea_gitlab_exporter/utils.py | 3 + 4 files changed, 101 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 069d0e0..987bd65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ + "click", "prettytable", "requests", ] @@ -26,7 +27,7 @@ dependencies = [ "Bug Tracker" = "https://git.evatt.ingenious.com.au/neillc/gitea-gitlab-exporter" [project.scripts] -gitlab2gitea = "gitea_gitlab_exporter:exporter" +gl2gt = "gitea_gitlab_exporter.cli:cli" [project.optional-dependencies] dev = [ diff --git a/src/gitea_gitlab_exporter/__init__.py b/src/gitea_gitlab_exporter/__init__.py index fa7e287..53f46b0 100644 --- a/src/gitea_gitlab_exporter/__init__.py +++ b/src/gitea_gitlab_exporter/__init__.py @@ -5,10 +5,3 @@ Copyright 2023 Neill Cox """ - -from .cli import cli_list_projects, parse_args - - -def exporter(): - args = parse_args() - cli_list_projects(args) diff --git a/src/gitea_gitlab_exporter/cli.py b/src/gitea_gitlab_exporter/cli.py index 0c60986..27ea8b9 100644 --- a/src/gitea_gitlab_exporter/cli.py +++ b/src/gitea_gitlab_exporter/cli.py @@ -4,7 +4,9 @@ import logging import os import sys +import click from prettytable import PrettyTable +from requests.exceptions import HTTPError from .api import get_project_details, get_user, list_projects @@ -76,12 +78,56 @@ def cli_get_user(args): print(json.dumps(user)) -def cli_list_projects(args): - logging.debug("cli_list_projects called") +class Context: + pass - projects = list_projects(args) - if args.format == "table": +@click.group() +@click.option("--format", default="json") +@click.option("--log-file", default="json") +@click.option( + "-t", + "--gitlab-token", + help=( + "A private access token to access GitLab with. If not specified " + "will use $GL_TOKEN. Required." + ), + envvar="GL2GT_GL_TOKEN", + required=True, +) +@click.option("--debug", is_flag=True) +@click.option("--log-file", default="gitlab2gitea.log") +@click.pass_context +def cli(ctx, format, gitlab_token, debug, log_file): + ctx.ensure_object(Context) + + ctx.obj.format = format + ctx.obj.gitlab_token = gitlab_token + ctx.obj.debug = debug + ctx.obj.log_file = log_file + + log_level = logging.INFO + if debug: + log_level = logging.DEBUG + + logging.basicConfig(filename=log_file, level=log_level) + + +@click.command() +@click.pass_context +def list_projects_click(ctx): + format = ctx.obj.format + + try: + projects = list_projects(ctx.obj) + except HTTPError as err: + if err.response.status_code == 401: + print("Invalid gitlab credentials") + sys.exit(2) + else: + raise + + if format == "table": tbl = PrettyTable() tbl.align = "l" tbl.field_names = [ @@ -100,7 +146,52 @@ def cli_list_projects(args): ) print(tbl) - elif args.format == "json": + elif format == "json": print(json.dumps(projects)) else: print(projects) + + +cli.add_command(list_projects_click) + + +@click.command() +@click.pass_context +def click_get_user(ctx): + logging.debug("cli_get_user called") + + user = get_user(ctx.obj) + + print(json.dumps(user)) + + +cli.add_command(click_get_user) + +# def cli_list_projects(args): +# logging.debug("cli_list_projects called") + +# projects = list_projects(args) + +# if args.format == "table": +# tbl = PrettyTable() +# tbl.align = "l" +# tbl.field_names = [ +# "ID", +# "Name", +# # "Description" +# ] + +# for row in projects: +# tbl.add_row( +# [ +# row["id"], +# row["name"], +# # row["description"][:50] if row["description"] else "" +# ] +# ) + +# print(tbl) +# elif args.format == "json": +# print(json.dumps(projects)) +# else: +# print(projects) diff --git a/src/gitea_gitlab_exporter/utils.py b/src/gitea_gitlab_exporter/utils.py index 45a54d4..d8291d6 100644 --- a/src/gitea_gitlab_exporter/utils.py +++ b/src/gitea_gitlab_exporter/utils.py @@ -13,6 +13,9 @@ def get(url, args): response = requests.get(url, headers={"PRIVATE-TOKEN": args.gitlab_token}) + # if not response.ok: + response.raise_for_status() + logging.debug("response.status: %s", response.status_code) logging.debug("body: %s", response.text) From 7c4e1c6d5fe88318eac21f3b11664d4c8ce44669 Mon Sep 17 00:00:00 2001 From: Neill Cox Date: Tue, 24 Oct 2023 13:13:28 +1100 Subject: [PATCH 4/5] :construction: remove commented code --- src/gitea_gitlab_exporter/cli.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/gitea_gitlab_exporter/cli.py b/src/gitea_gitlab_exporter/cli.py index 27ea8b9..526ce93 100644 --- a/src/gitea_gitlab_exporter/cli.py +++ b/src/gitea_gitlab_exporter/cli.py @@ -166,32 +166,3 @@ def click_get_user(ctx): cli.add_command(click_get_user) - -# def cli_list_projects(args): -# logging.debug("cli_list_projects called") - -# projects = list_projects(args) - -# if args.format == "table": -# tbl = PrettyTable() -# tbl.align = "l" -# tbl.field_names = [ -# "ID", -# "Name", -# # "Description" -# ] - -# for row in projects: -# tbl.add_row( -# [ -# row["id"], -# row["name"], -# # row["description"][:50] if row["description"] else "" -# ] -# ) - -# print(tbl) -# elif args.format == "json": -# print(json.dumps(projects)) -# else: -# print(projects) From 57dd12de070d1ca3d51b8d7dfab6edd6eb78798f Mon Sep 17 00:00:00 2001 From: Neill Cox Date: Tue, 24 Oct 2023 22:02:49 +1100 Subject: [PATCH 5/5] :construction: inital export with issues and wiki pages --- .pre-commit-config.yaml | 2 +- src/gitea_gitlab_exporter/api.py | 18 +++++++++++++ src/gitea_gitlab_exporter/cli.py | 43 +++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d63dcb..1b17c37 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: rev: 5.12.0 hooks: - id: isort - args: ["--profile", "black"] + args: ["--profile", "black", "-l", "79"] - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: diff --git a/src/gitea_gitlab_exporter/api.py b/src/gitea_gitlab_exporter/api.py index ef7e7d4..e965386 100644 --- a/src/gitea_gitlab_exporter/api.py +++ b/src/gitea_gitlab_exporter/api.py @@ -47,3 +47,21 @@ def get_project_details(args): raise KeyError(f"Project {args.project} not found") print(json.dumps(project)) + + +def get_issues(args): + project_id = args.project_id + url = gitlab_url( + f"/projects/{project_id}/issues?pagination=offset&per_page=500&" + ) + + response = get(url, args) + + return response + + +def get_wiki(args): + url = gitlab_url(f"/projects/{args.project_id}/wikis?with_content=1") + response = get(url, args) + + return response diff --git a/src/gitea_gitlab_exporter/cli.py b/src/gitea_gitlab_exporter/cli.py index 526ce93..d15bbbf 100644 --- a/src/gitea_gitlab_exporter/cli.py +++ b/src/gitea_gitlab_exporter/cli.py @@ -8,7 +8,13 @@ import click from prettytable import PrettyTable from requests.exceptions import HTTPError -from .api import get_project_details, get_user, list_projects +from .api import ( + get_issues, + get_project_details, + get_user, + get_wiki, + list_projects, +) def parse_args(): @@ -166,3 +172,38 @@ def click_get_user(ctx): cli.add_command(click_get_user) + + +@click.command() +@click.option("--project", help="Project name or ID", required=True) +@click.pass_context +def export_project(ctx, project): + project_list = [ + prj + for prj in list_projects(ctx.obj) + if (prj["id"] == project or prj["name"] == project) + ] + + if len(project_list) > 1: + print("Multiple projects found.") + sys.exit(2) + + if len(project_list) < 1: + print("No matching projects") + sys.exit(2) + + project_details = project_list[0] + + ctx.obj.project_id = project_details["id"] + + issues = get_issues(ctx.obj) + + project_details["issues"] = issues + + wiki_pages = get_wiki(ctx.obj) + project_details["wiki_pages"] = wiki_pages + + print(json.dumps(project_details)) + + +cli.add_command(export_project)