-
Notifications
You must be signed in to change notification settings - Fork 38
Expand file tree
/
Copy pathgo
More file actions
executable file
·326 lines (258 loc) · 9.04 KB
/
go
File metadata and controls
executable file
·326 lines (258 loc) · 9.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
#!/bin/bash
set -e
SCRIPT_DIR=$(cd $(dirname $0) ; pwd -P)
VENV_DIR="${SCRIPT_DIR}/venvs/trailscraper-venv${VENV_POSTFIX}"
VERSIONS="3.10
3.11
3.12
3.13
3.14"
activate_venv() {
source "${VENV_DIR}/bin/activate"
}
die() {
echo "$1"
exit 1
}
goal_regenerate_iam_data_from_cloudonaut() {
tmp_dir=$(mktemp -d)
pushd ${tmp_dir} > /dev/null
git clone --depth 1 [email protected]:widdix/complete-aws-iam-reference.git
cd complete-aws-iam-reference/tools
node md2json.js | \
tee ${SCRIPT_DIR}/tests/iam-actions-from-cloudonaut.json | \
jq -r '.[] | .service+":"+.action' | \
sort | \
uniq > ${SCRIPT_DIR}/tests/iam-actions-from-cloudonaut.txt
popd > /dev/null
}
goal_regenerate_iam_data_from_policy_sim() {
curl https://raw.githubusercontent.com/rvedotrc/aws-iam-reference/master/all-actions.txt |\
sort | \
uniq > ${SCRIPT_DIR}/tests/iam-actions-from-policy-sim.txt
}
goal_merge_iam_data() {
cat ${SCRIPT_DIR}/tests/iam-actions-from-cloudonaut.txt \
${SCRIPT_DIR}/tests/iam-actions-from-policy-sim.txt | \
sort | uniq > ${SCRIPT_DIR}/tests/known-iam-actions.txt
}
goal_regenerate_iam_data() {
goal_regenerate_iam_data_from_cloudonaut
goal_regenerate_iam_data_from_policy_sim
goal_merge_iam_data
}
goal_unknown-actions() {
activate_venv
python3 "${SCRIPT_DIR}/tests/list_unknown_actions.py" > unknown_actions.txt
}
goal_in-version() {
python_version=$1
cmd="/code/go ${@:2}"
echo "=============== BEGIN Python ${python_version} ${cmd} ==============="
docker run -i \
-v $(pwd):/code \
-w /code \
-e VENV_POSTFIX=${python_version} \
ghcr.io/astral-sh/uv:python${python_version}-trixie-slim ${cmd}
echo "=============== END Python ${python_version} ${cmd} ==============="
}
goal_in-all-versions() {
if [ "${parallel}" == "parallel" ]; then
parallel --tag "./go in-version {} ${@}" ::: "$VERSIONS"
else
for version in ${VERSIONS}; do
goal_in-version ${version} ${@}
done
fi
}
create_venv() {
venv_dir="$1"
python3 --version
which python3
uv venv "${venv_dir}"
}
goal_test() {
# make build not fail on travisci when working with boto and moto... https://github.com/spulec/moto/issues/1771
export BOTO_CONFIG=/dev/null
export AWS_SECRET_ACCESS_KEY=foobar_secret
export AWS_ACCESS_KEY_ID=foobar_key
pushd "${SCRIPT_DIR}" > /dev/null
activate_venv
pytest
popd > /dev/null
}
goal_build-artifacts() {
uv build
}
goal_smoketest-artifacts() {
local last_python_version=$(echo "${VERSIONS}" | sort | tail -n 1)
local python_version="${PYTHON_VERSION:-${last_python_version}}"
echo "Running Python ${python_version} in clean docker container..."
pushd "${SCRIPT_DIR}" > /dev/null
docker run -i -v $(pwd):/app python:${python_version} bash <<-EOF
pip install /app/dist/*.whl
trailscraper --version
cat /app/tests/smoke/policy-to-guess.json | trailscraper guess
cat /app/tests/smoke/events-for-smoke-test.json | trailscraper generate
EOF
popd > /dev/null
}
goal_build-docker() {
local last_python_version=$(echo "${VERSIONS}" | sort | tail -n 1)
local python_version="${PYTHON_VERSION:-${last_python_version}}"
docker build --build-arg PYTHON_VERSION=${python_version} -t ghcr.io/flosell/trailscraper:latest .
}
goal_smoketest-docker() {
cd ${SCRIPT_DIR}
docker run --rm -i ghcr.io/flosell/trailscraper:latest --version
cat ./tests/smoke/policy-to-guess.json | docker run --rm -i ghcr.io/flosell/trailscraper:latest guess
cat ./tests/smoke/events-for-smoke-test.json | docker run --rm -i ghcr.io/flosell/trailscraper:latest generate
}
goal_check() {
pushd "${SCRIPT_DIR}" > /dev/null
activate_venv
pylint trailscraper
popd > /dev/null
}
goal_trailscraper() {
uv run trailscraper "$@"
}
goal_setup() {
if [ ! -d "${VENV_DIR}" ]; then
create_venv "${VENV_DIR}"
fi
pushd "${SCRIPT_DIR}" > /dev/null
activate_venv
uv pip install -e ".[dev]"
popd > /dev/null
}
goal_clean() {
rm -rf "${VENV_DIR}"
pushd "${SCRIPT_DIR}" > /dev/null
set +e # It's OK if some of the things we want to delete aren't there
rm -fr build/
rm -fr dist/
rm -fr .eggs/
find . -name '*.egg-info' -exec rm -vfr {} +
find . -name '*.egg' -exec rm -vf {} +
find . -name '*.pyc' -exec rm -vf {} +
find . -name '*.pyo' -exec rm -vf {} +
find . -name '*~' -exec rm -vf {} +
find . -name '__pycache__' -exec rm -vfr {} +
set -e
popd > /dev/null
}
goal_create_github_release() {
# make sure you have the following:
# https://github.com/mtdowling/chag
# $GITHUB_TOKEN is set
pushd "${SCRIPT_DIR}" > /dev/null
VERSION=$(chag latest)
CHANGELOG=$(chag contents)
USER="flosell"
REPO="trailscraper"
echo "Publishing Release to GitHub: "
echo "Version ${VERSION}"
echo "${CHANGELOG}"
echo
gh release create ${VERSION} \
--notes "${CHANGELOG}" \
--title ${VERSION}
echo "Published release on GitHub"
popd > /dev/null
}
goal_tag_version() {
VERSION=$(chag latest)
git tag -s ${VERSION} -m "Release ${VERSION}"
git push --tags
}
goal_bump_version() {
part=${1:-patch}
activate_venv
bumpversion ${part}
}
goal_bump-homebrew-release() {
export HOMEBREW_GITHUB_API_TOKEN="$(gh auth token)" # homebrew often uses a readonly token, set the one already used for release instead
local version="$1"
brew bump-formula-pr --verbose \
--version "${version}" \
trailscraper
}
goal_create_github_container_registry_release() {
local version="$1"
echo $GITHUB_TOKEN | docker login ghcr.io -u flosell --password-stdin
docker tag ghcr.io/flosell/trailscraper:latest ghcr.io/flosell/trailscraper:${version}
docker push ghcr.io/flosell/trailscraper:latest
docker push ghcr.io/flosell/trailscraper:${version}
}
goal_release() {
echo "Checking GitHub Login..."
gh auth status > /dev/null || die "Please login to GitHub using 'gh auth login'"
gh auth status | grep -E 'Token scopes.*workflow' || die "Please ensure the GitHub token has the 'workflow' scope using 'gh auth refresh -s workflow'"
gh auth status | grep -E 'Token scopes.*write:packages' || die "Please ensure the GitHub token has the 'write:packages' scope using 'gh auth refresh -s write:packages'"
if [ -z "$PYPI_TEST_TOKEN" ]; then
echo "PYPI_TEST_TOKEN is not set"
exit 1
fi
if [ -z "$PYPI_TOKEN" ]; then
echo "PYPI_TOKEN is not set"
exit 1
fi
export GITHUB_TOKEN=$(gh auth token)
VERSION=$(chag latest)
echo "Version to release is ${VERSION}."
echo
echo "A few things to check:"
echo "* Is this really the version you want to release? Have there been major changes that require a different version?"
echo "* Are you sure that CHANGELOG.md, pyproject.toml and __init__.py all show this version?"
echo
(cd ${SCRIPT_DIR} && grep ${VERSION} pyproject.toml \
CHANGELOG.md \
trailscraper/__init__.py | sed 's/^/ /')
echo
echo "Hit ENTER if you are ok and want to continue"
read
goal_test
goal_check
goal_build-artifacts
goal_smoketest-artifacts
UV_PUBLISH_TOKEN="${PYPI_TEST_TOKEN}" uv publish --publish-url https://test.pypi.org/legacy/
echo "Published to Test PyPI, verify the package and hit ENTER to continue"
echo "You can install the package from Test PyPi using 'pip install --index https://test.pypi.org/simple trailscraper==0.10.0'"
read
goal_tag_version
goal_create_github_release
UV_PUBLISH_TOKEN="${PYPI_TOKEN}" uv publish
goal_build-docker
goal_smoketest-docker
goal_create_github_container_registry_release ${VERSION}
goal_bump_version
git push
}
goal_push() {
goal_test
goal_check
git push
}
if type -t "goal_$1" &>/dev/null; then
goal_$1 "${@:2}"
else
echo "usage: $0 <goal>
goal:
setup -- set up development environment
test -- run all functional tests
check -- run all style checks
build-artifacts -- build artifacts that can be installed with pip (wheel/sdist)
smoketest-artifacts -- run a smoke-test after installing in a clean environment
build-docker -- build a docker image
smoketest-docker -- run a smoke-test after building a clean docker container
trailscraper -- call the current development state
in-version -- run a go-command in a particular version of python
in-all-versions -- run a go-command in all supported versions of python
release -- create and publish a new release
bump_version -- bump version
regenerate_iam_data -- regenerate list of known iam actions
unknown-actions -- regenerate list of unknown actions
"
exit 1
fi