From 9f693416355d8592e3ce68196d507ff684e0e34c Mon Sep 17 00:00:00 2001
From: root <root@mbs.cclinux.org>
Date: Fri, 14 May 2021 15:06:39 +0800
Subject: [PATCH] The first modification, mainly configuration and user
 authentication changes.

---
 d.log                                       |  1 +
 distrobuild/bootstrap/__init__.py           | 10 ++--
 distrobuild/common/__init__.py              | 10 +---
 distrobuild/routes/batches.py               |  4 +-
 distrobuild/routes/imports.py               |  4 +-
 distrobuild/settings.py                     | 54 +++++++++++----------
 distrobuild_scheduler/__init__.py           |  1 +
 distrobuild_scheduler/import_package.py     | 22 +++++++--
 requirements.txt                            |  4 +-
 requirements.txt.bak                        | 20 ++++++++
 ui/babel.config.js                          |  3 +-
 ui/public/index.html                        | 12 ++---
 ui/src/components/BuildsTable.tsx           |  4 +-
 ui/src/components/PaginatedResourceList.tsx |  4 +-
 ui/webpack.config.js                        | 10 ++--
 15 files changed, 98 insertions(+), 65 deletions(-)
 create mode 100644 d.log
 create mode 100644 requirements.txt.bak

diff --git a/d.log b/d.log
new file mode 100644
index 0000000..ec84c64
--- /dev/null
+++ b/d.log
@@ -0,0 +1 @@
+INFO:     127.0.0.1:39900 - "GET / HTTP/1.1" 500 Internal Server Error
diff --git a/distrobuild/bootstrap/__init__.py b/distrobuild/bootstrap/__init__.py
index 5ad2787..ce1fe2a 100644
--- a/distrobuild/bootstrap/__init__.py
+++ b/distrobuild/bootstrap/__init__.py
@@ -35,7 +35,7 @@ async def _internal_create_package(**kwargs) -> Package:
 
 
 async def process_repo_dump(repo: Repo, responsible_username: str) -> None:
-    with open(f"/tmp/{repo}_ALL.txt", "r") as f:
+    with open(f"/root/distrobuild-data/distrobuild-extra/{repo}_ALL.txt", "r") as f:
         lines = f.readlines()
         for line in lines:
             package_name = line.strip()
@@ -58,18 +58,22 @@ async def process_repo_dump(repo: Repo, responsible_username: str) -> None:
 
 
 async def process_module_dump(responsible_username: str) -> None:
-    f = open("/tmp/modules.json")
+    f = open("/root/distrobuild-data/distrobuild-extra/modules.json")
     module_list = json.loads(f.read())
     f.close()
 
     for module in module_list.keys():
+        print("module:     "+module)
         package_name = module.strip()
+        print("package:      "+package_name)
         existing_package = await Package.filter(name=package_name, repo__not=Repo.MODULAR_CANDIDATE).get_or_none()
         if existing_package:
+            print("已经存在package")
             if not existing_package.is_module and existing_package.repo != Repo.MODULAR_CANDIDATE:
                 existing_package.is_module = True
                 await existing_package.save()
         else:
+            print(package_name+"  不存在时，内部创建。")
             existing_package = await _internal_create_package(name=package_name,
                                                               is_module=True,
                                                               responsible_username=responsible_username)
@@ -89,6 +93,6 @@ async def process_module_dump(responsible_username: str) -> None:
             # date before bootstrapping the module list
             if not m_package:
                 continue
-
+            print("save package!!")
             await m_package.save()
             await PackageModule.create(package_id=m_package.id, module_parent_package_id=existing_package.id)
diff --git a/distrobuild/common/__init__.py b/distrobuild/common/__init__.py
index 4fcd1e7..b129ca2 100644
--- a/distrobuild/common/__init__.py
+++ b/distrobuild/common/__init__.py
@@ -93,13 +93,5 @@ async def batch_list_check(packages):
 
 
 def get_user(request: Request) -> dict:
-    user = request.session.get("user")
-    token = request.session.get("token")
-    if not user or not token:
-        if request.session.get("token"):
-            request.session.pop("token")
-        if request.session.get("user"):
-            request.session.pop("user")
-        raise HTTPException(401, "not authenticated")
-
+    user = {"preferred_username":"james"}
     return user
diff --git a/distrobuild/routes/batches.py b/distrobuild/routes/batches.py
index 9ec46cc..c2a8339 100644
--- a/distrobuild/routes/batches.py
+++ b/distrobuild/routes/batches.py
@@ -62,8 +62,8 @@ async def list_batch_imports():
 
 @router.post("/imports/", response_model=NewBatchResponse)
 async def batch_import_package(request: Request, body: BatchImportRequest):
-    get_user(request)
-
+   # get_user(request)
+    print("................")
     if body.should_precheck:
         await batch_list_check(body.packages)
 
diff --git a/distrobuild/routes/imports.py b/distrobuild/routes/imports.py
index d776d6e..7ed88a2 100644
--- a/distrobuild/routes/imports.py
+++ b/distrobuild/routes/imports.py
@@ -87,7 +87,7 @@ async def cancel_import(request: Request, import_id: int):
 @router.post("/", status_code=202)
 async def import_package_route(request: Request, body: Dict[str, ImportRequest], batch_import_id: Optional[int] = None):
     user = get_user(request)
-
+    print(user["preferred_username"])
     filters = gen_body_filters(body)
     package = await Package.filter(**filters).get_or_none()
     if not package:
@@ -98,5 +98,5 @@ async def import_package_route(request: Request, body: Dict[str, ImportRequest],
 
     build_order = await create_import_order(package, user["preferred_username"], batch_import_id)
     await import_package_task(build_order[0][0], build_order[0][1], build_order[1:])
-
+    print("import_package_task------------->"+str(build_order))
     return {}
diff --git a/distrobuild/settings.py b/distrobuild/settings.py
index 96c1aef..3ce202a 100644
--- a/distrobuild/settings.py
+++ b/distrobuild/settings.py
@@ -23,55 +23,57 @@ from pydantic import BaseSettings
 
 
 class Settings(BaseSettings):
-    bugs_api_key: str
-    gitlab_api_key: str
-    session_secret: str
-    message_secret: str
-    tag_prefix: str = "rocky"
+#    bugs_api_key: str
+    gitlab_api_key: str = "N6Xxs3CNPk4KpjwqgkPE"
+    session_secret: str = "session_secret"
+    message_secret: str = "jEtNJSFL1QBui7gA9H53Y-QJ59zKEAA_N7kmxRifQnQ="
+    tag_prefix: str = "circle"
     active_point_releases: List[str] = ["3", "4"]
     default_point_release: str = "3"
     redis_url: str = "redis://localhost"
-    database_url: str = "postgres://postgres:postgres@localhost/dbuild"
-    production: bool = False
-    debug: bool = False
+#    redis_url: str = "http://127.0.0.1:6379"
+    database_url: str = "postgres://distrobuild:distrobuild@localhost/distrobuild"
+    production: bool = True
+    debug: bool = True
 
     # srpmproc
-    gitlab_host: str
+    gitlab_host: str = "git.cclinux.org"
     repo_prefix: str = "/"
-    storage_addr: str
-    ssh_user: str = "git"
+    storage_addr: str = "https://git.cclinux.org/sources/"
+    ssh_user: str = "circleautomation"
     ssh_port: int = 22
     ssh_key_location: Optional[str]
     version: int = 8
     no_storage_download: bool = False
     no_storage_upload: bool = False
     import_logs_dir: str = "/tmp"
-    original_import_branch_prefix: str = "r"
-    original_rpm_prefix: str = "https://git.rockylinux.org/original/rpms"
-    original_module_prefix: str = "https://git.rockylinux.org/original/modules"
+    original_import_branch_prefix: str = "cc"
+    original_rpm_prefix: str = "https://git.cclinux.org/circle/rpms"
+    original_module_prefix: str = "https://git.cclinux.org/circle/modules"
 
     # mbs
-    mbs_url: str
+    mbs_url: str = "https://mbs.cclinux.org"
 
     # sigul
-    disable_sigul: bool = False
-    sigul_config_file: str = "/etc/distrobuild/sigul.conf"
-    sigul_passphrase: str
-    sigul_key_name: str = "signing"
-    sigul_key_id: str
+    disable_sigul: bool = True
+#    sigul_config_file: str = "/etc/distrobuild/sigul.conf"
+#    sigul_passphrase: str
+#    sigul_key_name: str = "signing"
+#    sigul_key_id: str
 
     # oidc
-    oidc_issuer: str
-    oidc_client_id: str
-    oidc_client_secret: str
-    oidc_scopes: str = "https://id.fedoraproject.org/scope/groups https://mbs.rockylinux.org/oidc/mbs-submit-build"
+    oidc_issuer: str = "https://accounts.cclinux.org/idp/openidc/"
+    oidc_client_id: str = "f3c9b61129114e62ba8672e5889957c5"
+    oidc_client_secret: str = "-HLpL1CSTWBpVU62DifveIMnX8B1GT4X"
+#    oidc_scopes: str = "https://id.fedoraproject.org/scope/groups https://mbs.cclinux.org/oidc/mbs-submit-build"
+    oidc_scopes: str = ""
     oidc_required_group: Optional[str]
 
     # appearance
-    distribution: str = "Rocky Linux"
+    distribution: str = "Circle Linux"
 
     # scheduler options
-    broker_url: str
+    broker_url: str = "amqp://guest:guest@127.0.0.1/"
     routing_key: str = "distrobuild"
     workers: int = 10
 
diff --git a/distrobuild_scheduler/__init__.py b/distrobuild_scheduler/__init__.py
index 0d1bb2b..c67ffdb 100644
--- a/distrobuild_scheduler/__init__.py
+++ b/distrobuild_scheduler/__init__.py
@@ -48,6 +48,7 @@ async def init_channel(loop) -> None:
 
 
 async def import_package_task(package_id: int, import_id: int, dependents: List[Tuple[int, int]]):
+    print("import_package_task :"+ str(package_id) +" ..............start")
     msg_body = {
         "message": "import_package",
         "package_id": package_id,
diff --git a/distrobuild_scheduler/import_package.py b/distrobuild_scheduler/import_package.py
index 4eeacf2..e702512 100644
--- a/distrobuild_scheduler/import_package.py
+++ b/distrobuild_scheduler/import_package.py
@@ -36,21 +36,26 @@ from distrobuild_scheduler.utils import gitlabify
 
 @atomic()
 async def do(package: Package, package_import: Import):
-    koji_session.packageListAdd(tags.base(), package.name, "distrobuild")
-
+    print('do 1')
+    print(str(tags.base())+","+str(package.name)+ ",james")
+    koji_session.packageListAdd(tags.base(), package.name, "james")
+    print('do 2')
     original = package.repo == Repo.ORIGINAL
     branch_commits = await srpmproc.import_project(package_import.id, package.name, package_import.module,
                                                    original=original)
+    print('do 3')
     for branch in branch_commits.keys():
         commit = branch_commits[branch]
         await ImportCommit.create(branch=branch, commit=commit, import__id=package_import.id)
-
+    print('do 4')
     package.last_import = datetime.datetime.now()
-
+    
     mode = "modules" if package_import.module else "rpms"
     project = gl.projects.get(f"{settings.repo_prefix[1:]}/{mode}/{gitlabify(package.name)}")
+    print('do 5')
     project.visibility = "public"
     project.save()
+    print('do 6')
 
 
 # noinspection DuplicatedCode
@@ -60,19 +65,26 @@ async def task(package_id: int, import_id: int, dependents: List[Tuple[int, int]
 
     if package_import.status != ImportStatus.CANCELLED:
         try:
+            print(str(package_id)+"  <-package_id ------------import_id->    "+str(import_id))
+            print('1')
             package_import.status = ImportStatus.IN_PROGRESS
+            print('2')
             await package_import.save()
-
+            print('3')
             await do(package, package_import)
         except Exception as e:
+            print('4')
             logger.error(e)
             package_import.status = ImportStatus.FAILED
             package.last_import = None
         else:
+            print('5')
             package_import.status = ImportStatus.SUCCEEDED
         finally:
+            print('6')
             await package_import.save()
             await package.save()
+            print('7')
 
     if len(dependents) > 0:
         await task(dependents[0][0], dependents[0][1], dependents[1:])
diff --git a/requirements.txt b/requirements.txt
index 4e2aa65..b7e75ec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 fastapi==0.63.0
-tortoise-orm[asyncpg]==0.16.17
+tortoise-orm[asyncpg]==0.17.2
 aerich==0.5.3
 starlette-context==0.3.1
 aiofiles==0.6.0
@@ -17,4 +17,4 @@ httpx==0.17.1
 cryptography==3.4.7
 boto3==1.17.49
 google-cloud-storage==1.37.1
-aioredis==1.3.1
\ No newline at end of file
+aioredis==1.3.1
diff --git a/requirements.txt.bak b/requirements.txt.bak
new file mode 100644
index 0000000..4e2aa65
--- /dev/null
+++ b/requirements.txt.bak
@@ -0,0 +1,20 @@
+fastapi==0.63.0
+tortoise-orm[asyncpg]==0.16.17
+aerich==0.5.3
+starlette-context==0.3.1
+aiofiles==0.6.0
+jinja2==2.11.3
+fastapi-pagination[tortoise]==0.7.0
+koji==1.24.0
+pydantic[dotenv]==1.8.1
+python-dotenv==0.17.0
+python-gitlab==2.6.0
+aio-pika==6.8.0
+python-multipart==0.0.5
+authlib==0.15.3
+itsdangerous==1.1.0
+httpx==0.17.1
+cryptography==3.4.7
+boto3==1.17.49
+google-cloud-storage==1.37.1
+aioredis==1.3.1
\ No newline at end of file
diff --git a/ui/babel.config.js b/ui/babel.config.js
index 94cc065..3e46b4d 100644
--- a/ui/babel.config.js
+++ b/ui/babel.config.js
@@ -8,7 +8,6 @@ module.exports = {
     '@babel/preset-env',
   ],
   plugins: [
-    '@babel/plugin-transform-runtime',
-    dev && 'react-refresh/babel',
+    '@babel/plugin-transform-runtime'
   ].filter(Boolean),
 };
diff --git a/ui/public/index.html b/ui/public/index.html
index 746a0e9..6c30528 100644
--- a/ui/public/index.html
+++ b/ui/public/index.html
@@ -9,14 +9,14 @@
 
   <script>
     window.SETTINGS = {
-      'kojiWeburl': '{{ koji_weburl}}',
-      'gitlabUrl': '{{ gitlab_url }}',
-      'repoPrefix': '{{ repo_prefix }}',
-      'distribution': '{{ distribution }}',
+      'kojiWeburl': 'https://koji.cclinux.org/koji/',
+      'gitlabUrl': 'https://git.cclinux.org/',
+      'repoPrefix': '/stage',
+      'distribution': 'Circle Linux',
     };
     window.STATE = {
-      'authenticated': {{ authenticated or 'false' }},
-      'fullName': '{{ full_name }}'
+      'authenticated':true,
+      'fullName': 'distrobuild'
     }
   </script>
 </body>
diff --git a/ui/src/components/BuildsTable.tsx b/ui/src/components/BuildsTable.tsx
index bf09368..c24f942 100644
--- a/ui/src/components/BuildsTable.tsx
+++ b/ui/src/components/BuildsTable.tsx
@@ -81,7 +81,7 @@ export const BuildsTable = (props: BuildsTableProps) => {
                 <TableCell className="space-x-4">
                   {item.koji_id && (
                     <a
-                      href={`https://kojidev.rockylinux.org/koji/taskinfo?taskID=${item.koji_id}`}
+                      href={`https://koji.cclinux.org/koji/taskinfo?taskID=${item.koji_id}`}
                       target="_blank"
                     >
                       Koji
@@ -89,7 +89,7 @@ export const BuildsTable = (props: BuildsTableProps) => {
                   )}
                   {item.mbs_id && (
                     <a
-                      href={`https://mbsstg.rockylinux.org/module-build-service/1/module-builds/${item.mbs_id}`}
+                      href={`https://mbs.cclinux.org/module-build-service/1/module-builds/${item.mbs_id}`}
                       target="_blank"
                     >
                       MBS
diff --git a/ui/src/components/PaginatedResourceList.tsx b/ui/src/components/PaginatedResourceList.tsx
index 0acfccb..4ac6ea2 100644
--- a/ui/src/components/PaginatedResourceList.tsx
+++ b/ui/src/components/PaginatedResourceList.tsx
@@ -211,7 +211,7 @@ export const PaginatedResourceList = <T extends DataTableRow>(
                               <>
                                 {pkg['koji_id'] && (
                                   <a
-                                    href={`https://kojidev.rockylinux.org/koji/taskinfo?taskID=${pkg['koji_id']}`}
+                                    href={`https://koji.cclinux.org/koji/taskinfo?taskID=${pkg['koji_id']}`}
                                     target="_blank"
                                   >
                                     Koji
@@ -219,7 +219,7 @@ export const PaginatedResourceList = <T extends DataTableRow>(
                                 )}
                                 {pkg['mbs_id'] && (
                                   <a
-                                    href={`https://mbsstg.rockylinux.org/module-build-service/1/module-builds/${pkg['mbs_id']}`}
+                                    href={`https://mbs.cclinux.org/module-build-service/1/module-builds/${pkg['mbs_id']}`}
                                     target="_blank"
                                   >
                                     MBS
diff --git a/ui/webpack.config.js b/ui/webpack.config.js
index c9dec0d..dd9e134 100644
--- a/ui/webpack.config.js
+++ b/ui/webpack.config.js
@@ -39,7 +39,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist'),
     filename: `files/distrobuild.[contenthash].js`,
     chunkFilename: `files/distrobuild.[id].chunk.[chunkhash].js`,
-    publicPath: dev ? 'http://localhost:8080/static/' : '/static/',
+    //publicPath: dev ? 'http://localhost:8080/static/' : '/static/',
   },
   resolve: {
     extensions: ['.js', '.jsx', '.ts', '.tsx', '.css'],
@@ -49,21 +49,23 @@ module.exports = {
       dry: dev,
     }),
     new HtmlWebpackPlugin({
-      filename: 'templates/index.html',
-      template: path.resolve(__dirname, 'public/index.html'),
+      filename: 'index.html',
+      template: path.resolve(__dirname, './public/index.html'),
     }),
     dev && new webpack.HotModuleReplacementPlugin(),
     dev && new ReactRefreshWebpackPlugin(),
     dev && new CompressionPlugin(),
   ].filter(Boolean),
   devServer: {
+    disableHostCheck: true,
     hot: true,
     liveReload: false,
     historyApiFallback: {
       index: '/',
     },
     writeToDisk: true,
-    allowedHosts: ['http://localhost:8090'],
+    //allowedHosts: ['http://localhost:8090'],
+    allowedHosts: ['https://distrobuild.cclinux.org'],
     headers: {
       'Access-Control-Allow-Origin': '*',
       'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, PATCH, OPTIONS',
-- 
2.27.0

