Skip to content

Commit 8c26231

Browse files
alex-ipalex-ip
committed
Multiple host configurations now supported.
1 parent eaa6fab commit 8c26231

5 files changed

Lines changed: 301 additions & 258 deletions

File tree

‎code/examples/create_guest_collections/README.md‎

Lines changed: 54 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -57,62 +57,64 @@ The path to a valid YAML or JSON guest collection configuration file needs to be
5757

5858
```yaml
5959
---
60-
globus_host: <globus hostname (e.g. c4e1bc.e229.gaccess.io)>
61-
high_assurance: <true | false>
62-
storage_gateways:
63-
- mapped_collections:
64-
- guest_collections:
65-
- collection_base_path: <directory path with leading />
66-
display_name: <guest collection name>
67-
permissions:
68-
- DATA_TYPE: access
69-
path: <relative path under guest collection>
70-
permissions: <r | rw>
71-
principal: <principal UUID>
72-
principal_type: <identity | group | all_authenticated_users>
73-
...
74-
public: <true | false>
75-
mapped_collection_id: <mapped collection UUID>"
60+
- globus_host: <globus hostname (e.g. c4e1bc.e229.gaccess.io)>
61+
high_assurance: <true | false>
62+
storage_gateways:
63+
- mapped_collections:
64+
- guest_collections:
65+
- collection_base_path: <directory path with leading />
66+
display_name: <guest collection name>
67+
permissions:
68+
- DATA_TYPE: access
69+
path: <relative path under guest collection>
70+
permissions: <r | rw>
71+
principal: <principal UUID>
72+
principal_type: <identity | group | all_authenticated_users>
73+
...
74+
public: <true | false>
75+
mapped_collection_id: <mapped collection UUID>"
76+
...
77+
storage_gateway_id: <storage gateway UUID>
7678
...
77-
storage_gateway_id: <storage gateway UUID>
78-
...
7979
```
8080

8181
### JSON
8282

8383
```json
84-
{
85-
"globus_host": "<globus hostname (e.g. c4e1bc.e229.gaccess.io)>",
86-
"high_assurance": <true | false>,
87-
"storage_gateways": [
88-
{
89-
"storage_gateway_id": "<storage gateway UUID>",
90-
"mapped_collections": [
91-
{
92-
"mapped_collection_id": "<mapped collection UUID>",
93-
"guest_collections": [
94-
{
95-
"collection_base_path": "<directory path with leading />",
96-
"display_name": "<guest collection name>",
97-
"public": <true | false>,
98-
"permissions": [
99-
{
100-
"DATA_TYPE": "access",
101-
"path": "<relative path under guest collection>",
102-
"permissions": "<r | rw>",
103-
"principal": "<principal UUID>",
104-
"principal_type": "<identity | group | all_authenticated_users>"
105-
},
106-
...
107-
]
108-
},
109-
...
110-
]
111-
},
112-
...
113-
]
114-
},
115-
...
116-
]
117-
}
84+
[
85+
{
86+
"globus_host": "<globus hostname (e.g. c4e1bc.e229.gaccess.io)>",
87+
"high_assurance": <true | false>,
88+
"storage_gateways": [
89+
{
90+
"storage_gateway_id": "<storage gateway UUID>",
91+
"mapped_collections": [
92+
{
93+
"mapped_collection_id": "<mapped collection UUID>",
94+
"guest_collections": [
95+
{
96+
"collection_base_path": "<directory path with leading />",
97+
"display_name": "<guest collection name>",
98+
"public": <true | false>,
99+
"permissions": [
100+
{
101+
"DATA_TYPE": "access",
102+
"path": "<relative path under guest collection>",
103+
"permissions": "<r | rw>",
104+
"principal": "<principal UUID>",
105+
"principal_type": "<identity | group | all_authenticated_users>"
106+
},
107+
...
108+
]
109+
},
110+
...
111+
]
112+
},
113+
...
114+
]
115+
},
116+
...
117+
]
118+
}
119+
]
118120
```

‎code/examples/create_guest_collections/create_guest_collections.py‎

Lines changed: 110 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -46,115 +46,117 @@ def manage_guest_collections(
4646
assert client_id and client_secret, 'Please specify client_id and client_secret'
4747

4848
client_app = ClientApp("guest_collection_manager", client_id=client_id, client_secret=client_secret)
49-
gcs_client = GCSClient(managed_guest_collections_config['globus_host'], app=client_app)
50-
51-
storage_gateways = list(gcs_client.get_storage_gateway_list())
52-
# print('storage_gateways:')
53-
# pprint(storage_gateways)
54-
55-
# Get all collections on the specified host
56-
collections = [
57-
collection
58-
for collection in gcs_client.get_collection_list()
59-
if collection['domain_name'].endswith(managed_guest_collections_config['globus_host'])
60-
]
61-
# print('collections:')
62-
# pprint(collections)
63-
64-
for storage_gateway_config in managed_guest_collections_config['storage_gateways']:
65-
storage_gateway_id = storage_gateway_config['storage_gateway_id']
66-
67-
assert storage_gateway_id in [storage_gateway['id'] for storage_gateway in storage_gateways], f'Invalid Storage Gateway ID {storage_gateway_id}'
68-
69-
ensure_user_credential(gcs_client, storage_gateway_id)
70-
71-
for mapped_collection_config in storage_gateway_config['mapped_collections']:
72-
mapped_collection_id = mapped_collection_config['mapped_collection_id']
73-
74-
assert mapped_collection_id in [mapped_collection['id'] for mapped_collection in collections if mapped_collection['collection_type'] == 'mapped'], \
75-
f'Invalid Mapped Collection ID {mapped_collection_id}'
76-
77-
if not storage_gateway_config.get('high_assurance'):
78-
attach_data_access_scope(gcs_client, mapped_collection_id)
79-
80-
transfer_client = TransferClient(app=client_app).add_app_data_access_scope(mapped_collection_id)
81-
82-
for guest_collection_config in mapped_collection_config['guest_collections']:
83-
84-
display_name = guest_collection_config['display_name']
85-
collection_base_path = guest_collection_config['collection_base_path']
86-
87-
create_directory(transfer_client, mapped_collection_id, collection_base_path)
88-
89-
matching_guest_collections = [
90-
collection for collection in collections
91-
if collection['collection_type'] == 'guest'
92-
and collection['storage_gateway_id'] == storage_gateway_id
93-
and collection['mapped_collection_id'] == mapped_collection_id
94-
and collection['display_name'] == display_name
95-
]
96-
97-
if matching_guest_collections: # Update existing guest collection
98-
print(f'Guest collection "{display_name}" already exists.')
99-
if len(matching_guest_collections) == 1:
100-
collection = matching_guest_collections[0]
101-
else:
102-
print(f'Unable to determine unique match for collection "{display_name}"')
103-
exit(1)
104-
105-
# Unable to update collection_base_path (immutable), so we can't change that
106-
collection_request = GuestCollectionDocument(
107-
**({key: value for key, value in guest_collection_config.items() if key not in ['collection_base_path', 'permissions']} |
108-
{'mapped_collection_id': mapped_collection_id})
109-
)
110-
111-
print(f'Updating existing guest collection "{display_name}" with collection ID {collection["id"]}')
112-
gcs_client.update_collection(collection['id'], collection_request)
113-
114-
else: # Create new guest collection
115-
collection_request = GuestCollectionDocument(
116-
**({key: value for key, value in guest_collection_config.items() if key not in ['permissions']} |
117-
{'mapped_collection_id': mapped_collection_id})
118-
)
119-
120-
new_guest_collection = gcs_client.create_collection(collection_request)
121-
# print('new_guest_collection:')
122-
# pprint(new_guest_collection)
123-
print(f'Created new guest collection "{display_name}" with collection ID {new_guest_collection["id"]}')
124-
125-
for permission_config in guest_collection_config['permissions']:
126-
# Create guest collection subdirectory if it doesn't already exist
127-
if permission_config['path'] != '/':
128-
create_directory(transfer_client, mapped_collection_id, collection_base_path+permission_config['path'])
129-
130-
try:
131-
result = transfer_client.add_endpoint_acl_rule(collection["id"], permission_config)
132-
print(f'Added "{permission_config["permissions"]}" permissions to "{permission_config["path"]}" for {permission_config["principal_type"]} {permission_config["principal"]}')
133-
except TransferAPIError as e:
134-
if e.code == 'Exists':
135-
print(f'Permissions already exist to "{permission_config["path"]}" for {permission_config["principal_type"]} {permission_config["principal"]}')
49+
50+
for host_guest_collection_config in managed_guest_collections_config:
51+
gcs_client = GCSClient(host_guest_collection_config['globus_host'], app=client_app)
52+
53+
storage_gateways = list(gcs_client.get_storage_gateway_list())
54+
# print('storage_gateways:')
55+
# pprint(storage_gateways)
56+
57+
# Get all collections on the specified host
58+
collections = [
59+
collection
60+
for collection in gcs_client.get_collection_list()
61+
if collection['domain_name'].endswith(host_guest_collection_config['globus_host'])
62+
]
63+
# print('collections:')
64+
# pprint(collections)
65+
66+
for storage_gateway_config in host_guest_collection_config['storage_gateways']:
67+
storage_gateway_id = storage_gateway_config['storage_gateway_id']
68+
69+
assert storage_gateway_id in [storage_gateway['id'] for storage_gateway in storage_gateways], f'Invalid Storage Gateway ID {storage_gateway_id}'
70+
71+
ensure_user_credential(gcs_client, storage_gateway_id)
72+
73+
for mapped_collection_config in storage_gateway_config['mapped_collections']:
74+
mapped_collection_id = mapped_collection_config['mapped_collection_id']
75+
76+
assert mapped_collection_id in [mapped_collection['id'] for mapped_collection in collections if mapped_collection['collection_type'] == 'mapped'], \
77+
f'Invalid Mapped Collection ID {mapped_collection_id}'
78+
79+
if not storage_gateway_config.get('high_assurance'):
80+
attach_data_access_scope(gcs_client, mapped_collection_id)
81+
82+
transfer_client = TransferClient(app=client_app).add_app_data_access_scope(mapped_collection_id)
83+
84+
for host_guest_collection_config in mapped_collection_config['guest_collections']:
85+
86+
display_name = host_guest_collection_config['display_name']
87+
collection_base_path = host_guest_collection_config['collection_base_path']
88+
89+
create_directory(transfer_client, mapped_collection_id, collection_base_path)
90+
91+
matching_guest_collections = [
92+
collection for collection in collections
93+
if collection['collection_type'] == 'guest'
94+
and collection['storage_gateway_id'] == storage_gateway_id
95+
and collection['mapped_collection_id'] == mapped_collection_id
96+
and collection['display_name'] == display_name
97+
]
98+
99+
if matching_guest_collections: # Update existing guest collection
100+
print(f'Guest collection "{display_name}" already exists.')
101+
if len(matching_guest_collections) == 1:
102+
collection = matching_guest_collections[0]
136103
else:
137-
raise e
138-
139-
if purge_guest_collections:
140-
for existing_guest_collection in [
141-
collection
142-
for collection in collections
143-
if collection['collection_type'] == 'guest'
144-
and collection['mapped_collection_id'] == mapped_collection_id
145-
]:
146-
if existing_guest_collection['display_name'] not in [
147-
guest_collection_config['display_name']
148-
for guest_collection_config in mapped_collection_config['guest_collections']
149-
]:
150-
try:
151-
print(f"Deleting collection {existing_guest_collection['display_name']} (ID: {existing_guest_collection['id']})...")
152-
response = gcs_client.delete_collection(existing_guest_collection['id'])
153-
print(f"{response['code']}: {response['message']}")
154-
except GCSAPIError as e:
155-
print(f"Error deleting collection {existing_guest_collection['display_name']}: {e.code} - {e.message}")
156-
except Exception as e:
157-
print(f"An unexpected error occurred: {e}")
104+
print(f'Unable to determine unique match for collection "{display_name}"')
105+
exit(1)
106+
107+
# Unable to update collection_base_path (immutable), so we can't change that
108+
collection_request = GuestCollectionDocument(
109+
**({key: value for key, value in host_guest_collection_config.items() if key not in ['collection_base_path', 'permissions']} |
110+
{'mapped_collection_id': mapped_collection_id})
111+
)
112+
113+
print(f'Updating existing guest collection "{display_name}" with collection ID {collection["id"]}')
114+
gcs_client.update_collection(collection['id'], collection_request)
115+
116+
else: # Create new guest collection
117+
collection_request = GuestCollectionDocument(
118+
**({key: value for key, value in host_guest_collection_config.items() if key not in ['permissions']} |
119+
{'mapped_collection_id': mapped_collection_id})
120+
)
121+
122+
new_guest_collection = gcs_client.create_collection(collection_request)
123+
# print('new_guest_collection:')
124+
# pprint(new_guest_collection)
125+
print(f'Created new guest collection "{display_name}" with collection ID {new_guest_collection["id"]}')
126+
127+
for permission_config in host_guest_collection_config['permissions']:
128+
# Create guest collection subdirectory if it doesn't already exist
129+
if permission_config['path'] != '/':
130+
create_directory(transfer_client, mapped_collection_id, collection_base_path+permission_config['path'])
131+
132+
try:
133+
result = transfer_client.add_endpoint_acl_rule(collection["id"], permission_config)
134+
print(f'Added "{permission_config["permissions"]}" permissions to "{permission_config["path"]}" for {permission_config["principal_type"]} {permission_config["principal"]}')
135+
except TransferAPIError as e:
136+
if e.code == 'Exists':
137+
print(f'Permissions already exist to "{permission_config["path"]}" for {permission_config["principal_type"]} {permission_config["principal"]}')
138+
else:
139+
raise e
140+
141+
if purge_guest_collections:
142+
for existing_guest_collection in [
143+
collection
144+
for collection in collections
145+
if collection['collection_type'] == 'guest'
146+
and collection['mapped_collection_id'] == mapped_collection_id
147+
]:
148+
if existing_guest_collection['display_name'] not in [
149+
host_guest_collection_config['display_name']
150+
for host_guest_collection_config in mapped_collection_config['guest_collections']
151+
]:
152+
try:
153+
print(f"Deleting collection {existing_guest_collection['display_name']} (ID: {existing_guest_collection['id']})...")
154+
response = gcs_client.delete_collection(existing_guest_collection['id'])
155+
print(f"{response['code']}: {response['message']}")
156+
except GCSAPIError as e:
157+
print(f"Error deleting collection {existing_guest_collection['display_name']}: {e.code} - {e.message}")
158+
except Exception as e:
159+
print(f"An unexpected error occurred: {e}")
158160

159161

160162
def create_directory(

0 commit comments

Comments
 (0)
⚡