Skip to content

Commit 1861961

Browse files
steelecookesteelecooke
authored
Merge pull request #2 from AARNet/eres-541
Eres 541 - All changes for Advanced Globus Workshop
2 parents 1548a33 + 4684dba commit 1861961

22 files changed

Lines changed: 877 additions & 21 deletions

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,6 @@ Gemfile.lock
33
_site/
44
code/examples/globus_ansible/roles/globus/defaults/main.yml
55
code/examples/globus_ansible/inventory/host_vars/globus-test-host.yml
6-
code/examples/globus_ansible/inventory/all.yml
6+
code/examples/globus_ansible/inventory/all.yml
7+
globus-sdk-python/
8+
temp/

‎code/examples/globus_ansible/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ Please follow the instructions at the [Globus Automated Endpoint Deployment Guid
1111

1212
Additional information is available at [Globus How To Use Application Credentials or Service Accounts to Automate Data Transfer](https://docs.globus.org/guides/recipes/automate-with-service-account/).
1313

14+
NB: You will need to make sure that your service user is a valid subscription manager if you have a non-empty `globus_subscription_id` value.
15+
1416
### SSH keys
1517
You will also need to have a valid SSH public key installed in authorized_hosts on the target machine for the remote Ansible user, and the private key accessible on the Ansible host with this repository.
1618

‎code/examples/globus_ansible/roles/globus/tasks/collection.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@
4646
when: globus_endpoint_status.subscription_id != None
4747
block:
4848
- name: Permit guest collections (Subscription only)
49-
ansible.builtin.command: "{{ globus_gcs_binary }} collection update --allow-guest-collections {{ collection_id }}"
49+
ansible.builtin.command: "{{ globus_gcs_binary }} collection update --allow-guest-collections {{ collection_details.id }}"
5050

5151
- name: Get new collection metadata
52-
ansible.builtin.command: "{{ globus_gcs_binary }} collection show {{ collection_id }} --format json"
52+
ansible.builtin.command: "{{ globus_gcs_binary }} collection show {{ collection_details.id }} --format json"
5353
changed_when: false
5454
register: _globus_cmd_output
5555

‎code/examples/globus_ansible/roles/globus/tasks/config.yml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,21 @@
1515
ansible.builtin.set_fact:
1616
globus_endpoint_status: "{{ _globus_cmd_output.stdout | from_json }}"
1717

18-
- name: Add Globus Endpoint to specified subscription
19-
ansible.builtin.command: "{{ globus_gcs_binary }} endpoint set-subscription-id {{ globus_subscription_id }}"
18+
- name: Add Globus Endpoint to specified subscription and update endpoint status
2019
when: globus_endpoint_status.subscription_id == None and globus_subscription_id != ""
21-
changed_when: true
20+
block:
21+
- name: Add Globus Endpoint to specified subscription
22+
ansible.builtin.command: "{{ globus_gcs_binary }} endpoint set-subscription-id {{ globus_subscription_id }}"
23+
changed_when: true
24+
25+
- name: Reheck Globus Endpoint status
26+
ansible.builtin.command: "{{ globus_gcs_binary }} endpoint show --format=json"
27+
changed_when: false
28+
register: _globus_cmd_output
29+
30+
- name: Update globus_endpoint_status fact
31+
ansible.builtin.set_fact:
32+
globus_endpoint_status: "{{ _globus_cmd_output.stdout | from_json }}"
2233

2334
- name: Check Globus Endpoint roles
2435
ansible.builtin.command: "{{ globus_gcs_binary }} endpoint role list --format json"

‎globus-community-australasia/workshops/advanced_globus_workshop/README.md‎

Lines changed: 384 additions & 15 deletions
Large diffs are not rendered by default.

‎globus-community-australasia/workshops/jupyter_notebooks/Create guest collections.ipynb‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "bd810913-f6a1-4c4b-8aa0-7471e01845ea",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"\"\"\"\n",
11+
"Script to automatically set up RO & RW guest collections on the host(s) specified in GLOBUS_HOSTS. A small directory of test files will be\n",
12+
"transferred into the RO guest collection\n",
13+
"There is logic to prevent the creation of duplicate guest collections, and the standard test files are copied into the RO guest collection.\n",
14+
"The script assumes that each endpoint has exactly one storage gateway and exactly one public mapped collection.\n",
15+
"Anyone with write permissions to the public mapped collection can run this script.\n",
16+
"\"\"\"\n",
17+
"\n",
18+
"import globus_sdk\n",
19+
"from globus_sdk import TransferClient, TransferAPIError\n",
20+
"from globus_sdk.scopes import TransferScopes\n",
21+
"from globus_sdk.globus_app import UserApp\n",
22+
"from pprint import pprint\n",
23+
"\n",
24+
"########################################################################################################\n",
25+
"# USER TO COMPLETE - Enter UserApp Client UUID - UserApp needs to be created first\n",
26+
"NATIVE_CLIENT_ID = \"xxxx\"\n",
27+
"\n",
28+
"# USER TO COMPLETE - Enter Endpoint hostname shown by \"globus-connect-server endpoint show\" command\n",
29+
"# Hostname is found in the \"GCS Manager URL\" value\n",
30+
"GLOBUS_HOSTS = [\n",
31+
" \"xxxx.gaccess.io\",\n",
32+
"]\n",
33+
"\n",
34+
"# USER TO COMPLETE - Enter Globus User UUID for write permissions\n",
35+
"GLOBUS_USER_UUID = \"xxxx\"\n",
36+
"########################################################################################################\n",
37+
"\n",
38+
"# AARNet Globus Endpoint NSW (ARTM) POSIX Gateway Public RO Guest Collection\n",
39+
"# https://app.globus.org/file-manager?origin_id=9e472d3a-ac18-42d0-bac8-3c9220801fbe&two_pane=true\n",
40+
"SRC_COLLECTION = \"9e472d3a-ac18-42d0-bac8-3c9220801fbe\"\n",
41+
"SRC_PATH = \"/standard_test_files/5MB-in-tiny-files\"\n",
42+
"DST_PATH = \"/5MB-in-tiny-files\"\n",
43+
"\n",
44+
"# # Could also use original files from CERN (slower)\n",
45+
"# # https://app.globus.org/file-manager/collections/722751ce-1264-43b8-9160-a9272f746d78\n",
46+
"# SRC_COLLECTION = \"722751ce-1264-43b8-9160-a9272f746d78\"\n",
47+
"# SRC_PATH = \"/5MB-in-tiny-files\"\n",
48+
"# DST_PATH = \"/5MB-in-tiny-files\"\n",
49+
"\n",
50+
"\n",
51+
"def main():\n",
52+
" user_app = UserApp(\"create-guest-collections\", client_id=NATIVE_CLIENT_ID)\n",
53+
" # pprint(user_app.__dict__)\n",
54+
" for globus_host in GLOBUS_HOSTS:\n",
55+
" gcs_client = globus_sdk.GCSClient(globus_host, app=user_app)\n",
56+
" # pprint(gcs_client.__dict__)\n",
57+
" endpoint_id = gcs_client._endpoint_client_id\n",
58+
"\n",
59+
" storage_gateways = list(gcs_client.get_storage_gateway_list())\n",
60+
" # pprint(storage_gateways)\n",
61+
"\n",
62+
" if len(storage_gateways) != 1:\n",
63+
" print(f\"Unable to determine unique storage gateway for {globus_info['hostname']}\")\n",
64+
" exit(1)\n",
65+
"\n",
66+
" storage_gateway_id = storage_gateways[0]['id']\n",
67+
"\n",
68+
" collections = list(gcs_client.get_collection_list())\n",
69+
" # pprint(collections)\n",
70+
"\n",
71+
" # Find public mapped collection (assumes only one)\n",
72+
" mapped_collections = [collection for collection in collections if collection['collection_type'] == 'mapped' and collection['public']]\n",
73+
" if len(storage_gateways) != 1:\n",
74+
" print(f\"Unable to determine unique public mapped collection for {globus_info['hostname']}\")\n",
75+
" exit(1)\n",
76+
"\n",
77+
" mapped_collection = mapped_collections[0]\n",
78+
"\n",
79+
" transfer_client = TransferClient(app=user_app).add_app_data_access_scope(mapped_collection['id'])\n",
80+
" # pprint(transfer_client.__dict__)\n",
81+
"\n",
82+
" guest_collections = [\n",
83+
" collection for collection in collections\n",
84+
" if collection['collection_type'] == 'guest'\n",
85+
" and collection['public']\n",
86+
" and collection['mapped_collection_id'] == mapped_collection['id']\n",
87+
" ]\n",
88+
"\n",
89+
" # pprint(guest_collections)\n",
90+
"\n",
91+
" # Remove this line if the mapped collection is high assurance\n",
92+
" attach_data_access_scope(gcs_client, mapped_collection['id'])\n",
93+
"\n",
94+
" ensure_user_credential(gcs_client, storage_gateway_id)\n",
95+
"\n",
96+
" for collection_type in ['Public RO Guest', 'Public RW Guest']:\n",
97+
" # Create display name and base path - this is a little ugly and should be improved\n",
98+
" display_name = mapped_collection['display_name'].replace('Public Mapped', collection_type)\n",
99+
" collection_base_path = f\"/{collection_type.replace(' ', '_')}_Collection\"\n",
100+
"\n",
101+
" try:\n",
102+
" response = transfer_client.operation_mkdir(mapped_collection['id'], collection_base_path)\n",
103+
" # pprint(response)\n",
104+
" print(f'Created directory \"{collection_base_path}\" in collection \"{mapped_collection[\"display_name\"]}\"')\n",
105+
" except TransferAPIError as e:\n",
106+
" # pprint(e.__dict__)\n",
107+
" if e.code == 'ExternalError.MkdirFailed.Exists':\n",
108+
" print(f'Directory \"{collection_base_path}\" already exists in collection \"{mapped_collection[\"display_name\"]}\"')\n",
109+
" else:\n",
110+
" raise e\n",
111+
"\n",
112+
" collection_request = globus_sdk.GuestCollectionDocument(\n",
113+
" public=True,\n",
114+
" collection_base_path=collection_base_path,\n",
115+
" display_name=display_name,\n",
116+
" mapped_collection_id=mapped_collection['id'],\n",
117+
" )\n",
118+
" # pprint(collection_request)\n",
119+
"\n",
120+
" found_collections = [\n",
121+
" collection for collection in collections \n",
122+
" if collection['collection_type'] == 'guest' \n",
123+
" and collection['public']\n",
124+
" and collection['display_name'] == display_name\n",
125+
" ]\n",
126+
"\n",
127+
" if found_collections:\n",
128+
" print(f'Public guest collection \"{display_name}\" already exists.')\n",
129+
" if len(found_collections) == 1:\n",
130+
" collection = found_collections[0]\n",
131+
" else:\n",
132+
" print(f'Unable to determine unique match for collection \"{display_name}\"')\n",
133+
" exit(1)\n",
134+
"\n",
135+
" # Unable to update collection_base_path (immutable), so we can't change that\n",
136+
" print(f'Updating existing guest collection \"{display_name}\" with collection ID {collection[\"id\"]}')\n",
137+
" gcs_client.update_collection(collection['id'], collection_request)\n",
138+
"\n",
139+
" else:\n",
140+
" collection = gcs_client.create_collection(collection_request)\n",
141+
" print(f'Created new guest collection \"{display_name}\" with collection ID {collection[\"id\"]}')\n",
142+
"\n",
143+
" # Add read permissions for all authenticated users on all collections\n",
144+
" rule_data = {\n",
145+
" 'DATA_TYPE': 'access',\n",
146+
" 'path': '/',\n",
147+
" 'permissions': 'r',\n",
148+
" 'principal': '',\n",
149+
" 'principal_type': 'all_authenticated_users',\n",
150+
" }\n",
151+
" try:\n",
152+
" result = transfer_client.add_endpoint_acl_rule(collection[\"id\"], rule_data)\n",
153+
" # pprint(result)\n",
154+
" print('Added \"r\" permissions for \"all_authenticated_users\"')\n",
155+
" except TransferAPIError as e:\n",
156+
" # pprint(e.__dict__)\n",
157+
" if e.code == 'Exists':\n",
158+
" print('Permissions already exist for \"all authenticated users\"')\n",
159+
" else:\n",
160+
" raise e\n",
161+
"\n",
162+
" # Add read/write permissions for specified user on RW collection\n",
163+
" # Note that this is actually superfluous if the UUID is for the same user who owns the collection\n",
164+
" if ' RW ' in collection_type:\n",
165+
" rule_data = {\n",
166+
" 'DATA_TYPE': 'access',\n",
167+
" 'path': '/',\n",
168+
" 'permissions': 'rw',\n",
169+
" 'principal': GLOBUS_USER_UUID,\n",
170+
" 'principal_type': 'identity',\n",
171+
" }\n",
172+
" try:\n",
173+
" result = transfer_client.add_endpoint_acl_rule(collection[\"id\"], rule_data)\n",
174+
" # pprint(result)\n",
175+
" print(f'Added \"rw\" permissions for user \"{GLOBUS_USER_UUID}\"')\n",
176+
" except TransferAPIError as e:\n",
177+
" # pprint(e.__dict__)\n",
178+
" if e.code == 'Exists':\n",
179+
" print(f'Permissions already exist for user \"{GLOBUS_USER_UUID}\"')\n",
180+
" else:\n",
181+
" raise e\n",
182+
"\n",
183+
" # Transfer standard test files into RO collection\n",
184+
" elif ' RO ' in collection_type:\n",
185+
" try:\n",
186+
" response = transfer_client.operation_mkdir(collection[\"id\"], DST_PATH) # Error if directory already exists\n",
187+
" # pprint(response)\n",
188+
" print(f'Created directory \"{DST_PATH}\" in collection \"{collection[\"display_name\"]}\"')\n",
189+
"\n",
190+
" transfer_request = globus_sdk.TransferData(\n",
191+
" source_endpoint=SRC_COLLECTION,\n",
192+
" destination_endpoint=collection[\"id\"],\n",
193+
" )\n",
194+
" transfer_request.add_item(SRC_PATH, DST_PATH)\n",
195+
"\n",
196+
" task = transfer_client.submit_transfer(transfer_request)\n",
197+
" print(f\"Submitted transfer of standard test files to {DST_PATH}. Task ID: {task['task_id']}.\")\n",
198+
" except TransferAPIError as e:\n",
199+
" # pprint(e.__dict__)\n",
200+
" if e.code == 'ExternalError.MkdirFailed.Exists':\n",
201+
" print(f'Directory \"{DST_PATH}\" already exists in collection \"{collection[\"display_name\"]}\"')\n",
202+
" else:\n",
203+
" raise e\n",
204+
"\n",
205+
"\n",
206+
"def attach_data_access_scope(gcs_client, collection_id):\n",
207+
" \"\"\"Compose and attach a ``data_access`` scope for the supplied collection\"\"\"\n",
208+
" endpoint_scopes = gcs_client.get_gcs_endpoint_scopes(gcs_client.endpoint_client_id)\n",
209+
" collection_scopes = gcs_client.get_gcs_collection_scopes(collection_id)\n",
210+
"\n",
211+
" manage_collections = globus_sdk.Scope(endpoint_scopes.manage_collections)\n",
212+
" data_access = globus_sdk.Scope(collection_scopes.data_access, optional=True)\n",
213+
"\n",
214+
" manage_collections.add_dependency(data_access)\n",
215+
"\n",
216+
" gcs_client.add_app_scope(manage_collections)\n",
217+
"\n",
218+
"\n",
219+
"def ensure_user_credential(gcs_client, storage_gateway_id):\n",
220+
" \"\"\"\n",
221+
" Ensure that the user has a user credential on the client.\n",
222+
" This is the mapping between Globus Auth (OAuth2) and the local system's permissions.\n",
223+
" \"\"\"\n",
224+
" # Depending on the endpoint & storage gateway, this request document may need to\n",
225+
" # include more complex information such as a local username.\n",
226+
" # Consult with the endpoint owner for more detailed info on user mappings and\n",
227+
" # other particular requirements.\n",
228+
" req = globus_sdk.UserCredentialDocument(storage_gateway_id=storage_gateway_id)\n",
229+
" try:\n",
230+
" gcs_client.create_user_credential(req)\n",
231+
" except globus_sdk.GCSAPIError as err:\n",
232+
" # A user credential already exists, no need to create it.\n",
233+
" if err.http_status != 409:\n",
234+
" raise\n",
235+
"\n",
236+
"if __name__ == '__main__':\n",
237+
" main()"
238+
]
239+
},
240+
{
241+
"cell_type": "code",
242+
"execution_count": null,
243+
"id": "27199d94-d278-4dd0-a8e2-deb88d5013c4",
244+
"metadata": {},
245+
"outputs": [],
246+
"source": []
247+
}
248+
],
249+
"metadata": {
250+
"kernelspec": {
251+
"display_name": "Python 3 (ipykernel)",
252+
"language": "python",
253+
"name": "python3"
254+
},
255+
"language_info": {
256+
"codemirror_mode": {
257+
"name": "ipython",
258+
"version": 3
259+
},
260+
"file_extension": ".py",
261+
"mimetype": "text/x-python",
262+
"name": "python",
263+
"nbconvert_exporter": "python",
264+
"pygments_lexer": "ipython3",
265+
"version": "3.11.11"
266+
}
267+
},
268+
"nbformat": 4,
269+
"nbformat_minor": 5
270+
}

0 commit comments

Comments
 (0)
⚡