Enable a containerized version of the ADD API endpoint

[CLOUDDST-28643]

Author: lipoja

iib/web/api_v1.py

@@ -44,8 +44,8 @@
 from iib.web.s3_utils import get_object_from_s3_bucket
 from botocore.response import StreamingBody
 from iib.web.utils import pagination_metadata, str_to_bool
-from iib.workers.tasks.build import (
-    handle_add_request,
+from iib.workers.tasks.build_containerized_add import (
+    handle_containerized_add_request,
 )
 from iib.workers.tasks.build_containerized_rm import handle_containerized_rm_request
 from iib.workers.tasks.build_add_deprecations import handle_add_deprecations_request
@@ -131,13 +131,9 @@ def _get_add_args(
         payload.get('binary_image'),
         payload.get('from_index'),
         payload.get('add_arches'),
-        payload.get('cnr_token'),
-        payload.get('organization'),
-        payload.get('force_backport'),
         overwrite_from_index,
         payload.get('overwrite_from_index_token'),
         request.distribution_scope,
-        flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue),
         flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'],
         payload.get('deprecation_list', []),
         payload.get('build_tags', []),
@@ -601,6 +597,9 @@ def add_bundles() -> Tuple[flask.Response, int]:
     if not isinstance(payload, dict):
         raise ValidationError('The input data must be a JSON object')
 
+    if not payload.get('from_index'):
+        raise ValidationError('The input "from_index" is required.')
+
     # Only run `_get_unique_bundles` if it is a list. If it's not, `from_json`
     # will raise an error to the user.
     if payload.get('bundles') and isinstance(payload['bundles'], list):
@@ -623,7 +622,7 @@ def add_bundles() -> Tuple[flask.Response, int]:
         args.append(current_user.username)
 
     try:
-        handle_add_request.apply_async(
+        handle_containerized_add_request.apply_async(
             args=args,
             link_error=error_callback,
             argsrepr=repr(safe_args),
@@ -1074,7 +1073,7 @@ def add_rm_batch() -> Tuple[flask.Response, int]:
         error_callback = failed_request_callback.s(request.id)
         try:
             if isinstance(request, RequestAdd):
-                handle_add_request.apply_async(
+                handle_containerized_add_request.apply_async(
                     args=args,
                     link_error=error_callback,
                     argsrepr=repr(safe_args),

iib/workers/config.py

@@ -94,6 +94,7 @@ class Config(object):
         'iib.workers.tasks.build_create_empty_index',
         'iib.workers.tasks.build_fbc_operations',
         'iib.workers.tasks.build_add_deprecations',
+        'iib.workers.tasks.build_containerized_add',
         'iib.workers.tasks.build_containerized_fbc_operations',
         'iib.workers.tasks.build_containerized_rm',
         'iib.workers.tasks.build_containerized_create_empty_index',

tests/test_web/test_api_v1.py

@@ -513,23 +513,39 @@ def test_get_build_logs_s3_configured(
             'The "overwrite_from_index_token" parameter must be a string',
         ),
         (
-            {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'cnr_token': True},
-            '"cnr_token" must be a string',
-        ),
-        (
-            {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'organization': True},
+            {
+                'from_index': 'pull:spec',
+                'bundles': ['some:thing'],
+                'binary_image': 'binary:image',
+                'organization': True,
+            },
             '"organization" must be a string',
         ),
         (
-            {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'force_backport': 'spam'},
+            {
+                'from_index': 'pull:spec',
+                'bundles': ['some:thing'],
+                'binary_image': 'binary:image',
+                'force_backport': 'spam',
+            },
             '"force_backport" must be a boolean',
         ),
         (
-            {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'graph_update_mode': 123},
+            {
+                'from_index': 'pull:spec',
+                'bundles': ['some:thing'],
+                'binary_image': 'binary:image',
+                'graph_update_mode': 123,
+            },
             '"graph_update_mode" must be a string',
         ),
         (
-            {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'graph_update_mode': 'Hi'},
+            {
+                'from_index': 'pull:spec',
+                'bundles': ['some:thing'],
+                'binary_image': 'binary:image',
+                'graph_update_mode': 'Hi',
+            },
             (
                 '"graph_update_mode" must be set to one of these: [\'replaces\', \'semver\''
                 ', \'semver-skippatch\']'
@@ -560,7 +576,7 @@ def test_add_bundles_overwrite_not_allowed(mock_smfsc, client, db):
     mock_smfsc.assert_not_called()
 
 
-@pytest.mark.parametrize('from_index', (None, 'some-random-index:v4.14', 'some-common-index:v4.15'))
+@pytest.mark.parametrize('from_index', ('some-random-index:v4.14', 'some-common-index:v4.15'))
 @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
 def test_add_bundles_graph_update_mode_not_allowed(
     mock_smfsc, app, client, auth_env, db, from_index
@@ -587,7 +603,7 @@ def test_add_bundles_graph_update_mode_not_allowed(
 
 @pytest.mark.parametrize('from_index', ('some-common-index:v4.15', 'another-common-index:v4.15'))
 @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
-@mock.patch('iib.web.api_v1.handle_add_request')
+@mock.patch('iib.web.api_v1.handle_containerized_add_request')
 def test_add_bundles_graph_update_mode_allowed(
     mock_har, mock_smfsc, app, client, auth_env, db, from_index
 ):
@@ -611,7 +627,7 @@ def test_add_bundles_graph_update_mode_allowed(
 @mock.patch('iib.web.api_v1.db.session')
 @mock.patch('iib.web.api_v1.flask.jsonify')
 @mock.patch('iib.web.api_v1.RequestAdd')
-@mock.patch('iib.web.api_v1.handle_add_request.apply_async')
+@mock.patch('iib.web.api_v1.handle_containerized_add_request.apply_async')
 @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
 def test_add_bundles_unique_bundles(mock_smfsc, mock_har, mock_radd, mock_fj, mock_dbs, client):
     data = {
@@ -690,15 +706,16 @@ def test_rm_operators_overwrite_not_allowed(mock_smfsc, client, db):
         ),
         (
             {'add_arches': ['s390x'], 'binary_image': 'binary:image'},
-            '"from_index" must be specified if no bundles are specified',
+            'The input "from_index" is required.',
         ),
-        ({'add_arches': ['s390x']}, '"from_index" must be specified if no bundles are specified'),
+        ({'add_arches': ['s390x']}, 'The input "from_index" is required.'),
         (
             {
                 'bundles': ['some:thing'],
                 'binary_image': 'binary:image',
                 'add_arches': ['s390x'],
                 'overwrite_from_index_token': 'username:password',
+                'from_index': 'pull:spec',
             },
             (
                 'The "overwrite_from_index" parameter is required when the '
@@ -758,6 +775,7 @@ def test_add_bundle_invalid_param(mock_smfsc, db, auth_env, client):
         'best_batsman': 'Virat Kohli',
         'binary_image': 'binary:image',
         'bundles': ['some:thing'],
+        'from_index': 'pull:spec',
     }
 
     rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env)
@@ -802,16 +820,6 @@ def test_rm_bundle_from_invalid_distribution_scope(mock_smfsc, db, auth_env, cli
     mock_smfsc.assert_not_called()
 
 
-@mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
-def test_add_bundle_from_index_and_add_arches_missing(mock_smfsc, db, auth_env, client):
-    data = {'bundles': ['some:thing'], 'binary_image': 'binary:image'}
-
-    rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env)
-    assert rv.status_code == 400
-    assert rv.json['error'] == 'One of "from_index" or "add_arches" must be specified'
-    mock_smfsc.assert_not_called()
-
-
 @pytest.mark.parametrize(
     (
         'binary_image',
@@ -823,14 +831,14 @@ def test_add_bundle_from_index_and_add_arches_missing(mock_smfsc, db, auth_env,
         'graph_update_mode',
     ),
     (
-        ('binary:image', False, None, ['some:thing'], None, None, None),
+        ('binary:image', False, None, ['some:thing'], 'some:thing', None, None),
         ('binary:image', False, None, ['some:thing'], 'some:thing', None, 'semver'),
         ('binary:image', False, None, [], 'some:thing', 'Prod', 'semver-skippatch'),
         ('scratch', True, 'username:password', ['some:thing'], 'some:thing', 'StagE', 'replaces'),
         ('scratch', True, 'username:password', [], 'some:thing', 'DeV', 'semver'),
     ),
 )
-@mock.patch('iib.web.api_v1.handle_add_request')
+@mock.patch('iib.web.api_v1.handle_containerized_add_request')
 @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
 def test_add_bundle_success(
     mock_smfsc,
@@ -851,8 +859,6 @@ def test_add_bundle_success(
     data = {
         'binary_image': binary_image,
         'add_arches': ['s390x'],
-        'organization': 'org',
-        'cnr_token': 'token',
         'overwrite_from_index': overwrite_from_index,
         'overwrite_from_index_token': overwrite_from_index_token,
         'from_index': from_index,
@@ -898,7 +904,7 @@ def test_add_bundle_success(
             'expiration': '2020-02-15T17:03:00Z',
         },
         'omps_operator_version': {},
-        'organization': 'org',
+        'organization': None,
         'state_history': [
             {
                 'state': 'in_progress',
@@ -918,31 +924,12 @@ def test_add_bundle_success(
     rv_json['logs']['expiration'] = '2020-02-15T17:03:00Z'
     assert rv.status_code == 201
     assert response_json == rv_json
-    assert 'cnr_token' not in rv_json
     assert 'token' not in mock_har.apply_async.call_args[1]['argsrepr']
-    assert '*****' in mock_har.apply_async.call_args[1]['argsrepr']
     mock_har.apply_async.assert_called_once()
     mock_smfsc.assert_called_once_with(mock.ANY, new_batch_msg=True)
 
 
-@pytest.mark.parametrize('force_backport', (False, True))
-@mock.patch('iib.web.api_v1.handle_add_request')
-def test_add_bundle_force_backport(mock_har, force_backport, db, auth_env, client):
-    data = {
-        'bundles': ['some:thing'],
-        'binary_image': 'binary:image',
-        'from_index': 'index:image',
-        'force_backport': force_backport,
-    }
-
-    rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env)
-    assert rv.status_code == 201
-    mock_har.apply_async.assert_called_once()
-    # Eigth element in args is the force_backport parameter
-    assert mock_har.apply_async.call_args[1]['args'][7] == force_backport
-
-
-@mock.patch('iib.web.api_v1.handle_add_request')
+@mock.patch('iib.web.api_v1.handle_containerized_add_request')
 @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
 def test_add_bundle_overwrite_token_redacted(mock_smfsc, mock_har, app, auth_env, client, db):
     token = 'username:password'
@@ -952,20 +939,20 @@ def test_add_bundle_overwrite_token_redacted(mock_smfsc, mock_har, app, auth_env
         'add_arches': ['amd64'],
         'overwrite_from_index': True,
         'overwrite_from_index_token': token,
+        'from_index': 'pull:spec',
     }
 
     rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env)
     rv_json = rv.json
     assert rv.status_code == 201
     mock_har.apply_async.assert_called_once()
     # Tenth to last element in args is the overwrite_from_index parameter
-    assert mock_har.apply_async.call_args[1]['args'][-11] is True
+    assert mock_har.apply_async.call_args[1]['args'][-10] is True
     # Ninth to last element in args is the overwrite_from_index_token parameter
-    assert mock_har.apply_async.call_args[1]['args'][-10] == token
+    assert mock_har.apply_async.call_args[1]['args'][-9] == token
     assert 'overwrite_from_index_token' not in rv_json
     assert token not in json.dumps(rv_json)
     assert token not in mock_har.apply_async.call_args[1]['argsrepr']
-    assert '*****' in mock_har.apply_async.call_args[1]['argsrepr']
 
 
 @pytest.mark.parametrize(
@@ -1002,13 +989,18 @@ def test_add_bundle_overwrite_token_redacted(mock_smfsc, mock_har, app, auth_env
         ({'[email protected]': 'Patriots'}, True, None),
     ),
 )
-@mock.patch('iib.web.api_v1.handle_add_request')
+@mock.patch('iib.web.api_v1.handle_containerized_add_request')
 @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
 def test_add_bundle_custom_user_queue(
     mock_smfsc, mock_har, app, auth_env, client, user_to_queue, overwrite_from_index, expected_queue
 ):
     app.config['IIB_USER_TO_QUEUE'] = user_to_queue
-    data = {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'add_arches': ['s390x']}
+    data = {
+        'bundles': ['some:thing'],
+        'binary_image': 'binary:image',
+        'add_arches': ['s390x'],
+        'from_index': 'pull:spec',
+    }
     if overwrite_from_index:
         data['from_index'] = 'index:image'
         data['overwrite_from_index'] = True
@@ -1820,7 +1812,7 @@ def test_regenerate_bundle_batch_invalid_input(payload, error_msg, app, auth_env
     assert rv.json == {'error': error_msg}
 
 
-@mock.patch('iib.web.api_v1.handle_add_request')
+@mock.patch('iib.web.api_v1.handle_containerized_add_request')
 @mock.patch('iib.web.api_v1.handle_containerized_rm_request')
 @mock.patch('iib.web.api_v1.messaging.send_messages_for_new_batch_of_requests')
 @mock.patch.dict('iib.web.api_v1.flask.current_app.config', {'IIB_INDEX_TO_GITLAB_PUSH_MAP': {}})
@@ -1834,8 +1826,6 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c
                 'binary_image': 'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5',
                 'from_index': 'registry-proxy/rh-osbs-stage/iib:v4.5',
                 'add_arches': ['amd64'],
-                'cnr_token': 'no_tom_brady_anymore',
-                'organization': 'hello-operator',
                 'overwrite_from_index': True,
                 'overwrite_from_index_token': 'some_token',
             },
@@ -1860,13 +1850,9 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c
                     'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5',
                     'registry-proxy/rh-osbs-stage/iib:v4.5',
                     ['amd64'],
-                    'no_tom_brady_anymore',
-                    'hello-operator',
-                    None,
                     True,
                     'some_token',
                     None,
-                    None,
                     {},
                     [],
                     [],
@@ -1875,11 +1861,10 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c
                     {},  # index_to_gitlab_push_map from config (empty in test)
                 ],
                 argsrepr=(
-                    "[['registry-proxy/rh-osbs/lgallett-bundle:v1.0-9'], "
-                    "1, 'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', "
-                    "'registry-proxy/rh-osbs-stage/iib:v4.5', ['amd64'], '*****', "
-                    "'hello-operator', None, True, '*****', None, None, {}, [], [], None, "
-                    "False, {}]"
+                    "[['registry-proxy/rh-osbs/lgallett-bundle:v1.0-9'], 1, "
+                    "'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', "
+                    "'registry-proxy/rh-osbs-stage/iib:v4.5', ['amd64'], True, '*****', "
+                    "None, {}, [], [], None, False, {}]"
                 ),
                 link_error=mock.ANY,
                 queue=None,
@@ -1904,8 +1889,8 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c
                 ],
                 argsrepr=(
                     "[['kiali-ossm'], 2, 'registry:8443/iib-build:11', "
-                    "'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5'"
-                    ", None, False, None, None, {}, [], {}]"
+                    "'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', "
+                    "None, False, None, None, {}, [], {}]"
                 ),
                 link_error=mock.ANY,
                 queue=None,