test: initial e2e publishing testing for deno

This commit is contained in:
alina 🌸 2024-04-23 13:20:42 +03:00
parent 78457fb158
commit 5caeff93a9
Signed by: teidesu
SSH key fingerprint: SHA256:uNeCpw6aTSU4aIObXLvHfLkDa82HWH9EiOj9AXOIRpI
22 changed files with 597 additions and 17 deletions

View file

@ -285,6 +285,12 @@ module.exports = {
'no-restricted-imports': 'off', 'no-restricted-imports': 'off',
'import/no-relative-packages': 'off', // common-internals is symlinked from node 'import/no-relative-packages': 'off', // common-internals is symlinked from node
} }
},
{
files: ['e2e/deno/**'],
rules: {
'import/no-unresolved': 'off',
}
} }
], ],
settings: { settings: {

View file

@ -1,3 +1,4 @@
**/node_modules **/node_modules
**/private **/private
**/dist **/dist
/e2e

3
e2e/deno/.dockerignore Normal file
View file

@ -0,0 +1,3 @@
/.jsr-data
/dist
/deno.lock

5
e2e/deno/.env.example Normal file
View file

@ -0,0 +1,5 @@
# obtain these values from my.telegram.org
API_ID=
API_HASH=
GITHUB_TOKEN=

3
e2e/deno/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
/.jsr-data
.env
/deno.lock

26
e2e/deno/Dockerfile.build Normal file
View file

@ -0,0 +1,26 @@
FROM denoland/deno:bin-1.42.4 as deno-bin
FROM node:20
WORKDIR /app
COPY --from=deno-bin /deno /bin/deno
# todo: remove once 1.42.5 is out
RUN deno upgrade --canary --version=2f5a6a8514ad8eadce1a0a9f1a7a419692e337ef
RUN corepack enable && \
corepack prepare pnpm@8.7.1 --activate
COPY ../.. /app/
RUN pnpm install --frozen-lockfile && \
pnpm -C packages/tl run gen-code
RUN apt update && apt install -y socat
ENV REGISTRY="http://jsr/"
ENV E2E="1"
ENV JSR="1"
ENV JSR_TOKEN="token"
ENTRYPOINT [ "node", "/app/scripts/publish.js" ]
CMD [ "all" ]

3
e2e/deno/Dockerfile.jsr Normal file
View file

@ -0,0 +1,3 @@
FROM ghcr.io/teidesu/jsr-api:latest
RUN apt update && apt install -y curl

10
e2e/deno/Dockerfile.test Normal file
View file

@ -0,0 +1,10 @@
FROM denoland/deno:1.42.4
WORKDIR /app
RUN apt update && apt install -y socat
COPY ./ /app/
ENV DOCKER="1"
ENTRYPOINT [ "./cli.sh", "run" ]

30
e2e/deno/README.md Normal file
View file

@ -0,0 +1,30 @@
# mtcute e2e tests (Deno edition)
This directory contains end-to-end tests for mtcute under Deno.
They are made for 2 purposes:
- Ensure published packages work as expected and can properly be imported
- Ensure that the library works with the actual Telegram API
To achieve the first goal, we use a local JSR instance container where we publish the package,
and then install it from there in another container
## Setting up
Before running the tests, you need to copy `.env.example` to `.env` and fill in the values
## Running tests
```bash
# first start a local jsr instance
./cli.sh start
# push all packages to the local registry
./cli.sh update
# pushing a particular package is not supported due to jsr limitations
# run the tests
./cli.sh run
# or in docker
./cli.sh run-docker
```

62
e2e/deno/cli.sh Executable file
View file

@ -0,0 +1,62 @@
#!/bin/bash
set -eau
method=$1
shift
case "$method" in
"start")
docker compose up -d --wait jsr
node ./init-server.js
;;
"update")
# unpublish all packages
rm -rf .jsr-data/gcs/modules/@mtcute/*
docker compose exec jsr-db psql registry -U user -c "delete from publishing_tasks;"
docker compose exec jsr-db psql registry -U user -c "delete from package_files;"
docker compose exec jsr-db psql registry -U user -c "delete from npm_tarballs;"
docker compose exec jsr-db psql registry -U user -c "delete from package_version_dependencies;"
docker compose exec jsr-db psql registry -U user -c "delete from package_versions;"
docker compose exec jsr-db psql registry -U user -c "delete from packages;"
# publish all packages
docker compose run --rm --build build all
# clear cache
rm -rf $(deno info --json | jq .denoDir -r)/deps
rm deno.lock
;;
"clean")
docker compose down
rm -rf .jsr-data
;;
"stop")
docker compose down
;;
"run")
source .env
if [ -n "$DOCKER" ]; then
# running behind a socat proxy seems to fix some of the docker networking issues (thx kamillaova)
socat TCP-LISTEN:4873,fork,reuseaddr TCP4:jsr:80 &
socat_pid=$!
trap "kill $socat_pid" EXIT
fi
export JSR_URL=http://localhost:4873
if [ -z "$@" ]; then
deno test -A tests/**/*.ts
else
deno test -A $@
fi
;;
"run-docker")
source .env
docker compose run --rm --build test $@
;;
*)
echo "Unknown command"
;;
esac

9
e2e/deno/deno.json Normal file
View file

@ -0,0 +1,9 @@
{
"imports": {
"@mtcute/web": "jsr:@mtcute/web@*",
"@mtcute/wasm": "jsr:@mtcute/wasm@*",
"@mtcute/tl": "jsr:@mtcute/tl@*",
"@mtcute/tl-runtime": "jsr:@mtcute/tl-runtime@*",
"@mtcute/core": "jsr:@mtcute/core@*"
}
}

View file

@ -0,0 +1,78 @@
version: "3"
services:
# jsr (based on https://github.com/teidesu/docker-images/blob/main/jsr/docker-compose.yaml)
jsr-db:
image: postgres:15
command: postgres -c 'max_connections=1000'
restart: always
environment:
POSTGRES_USER: user
POSTGRES_PASSWORD: password
POSTGRES_DB: registry
healthcheck:
test: "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"
interval: 5s
retries: 20
start_period: 5s
volumes:
- ./.jsr-data/db:/var/lib/postgresql/data
jsr-gcs:
image: fsouza/fake-gcs-server:latest
command: -scheme http -filesystem-root=/gcs-data -port 4080
volumes:
- ./.jsr-data/gcs:/gcs-data
jsr-api:
depends_on:
jsr-db:
condition: service_healthy
jsr-gcs:
condition: service_started
healthcheck:
test: "curl --fail http://localhost:8001/sitemap.xml || exit 1"
interval: 5s
retries: 20
start_period: 5s
build:
context: .
dockerfile: Dockerfile.jsr
environment:
- "DATABASE_URL=postgres://user:password@jsr-db/registry"
- "GITHUB_CLIENT_ID=fake"
- "GITHUB_CLIENT_SECRET=fake"
- "GCS_ENDPOINT=http://jsr-gcs:4080"
- "MODULES_BUCKET=modules"
- "PUBLISHING_BUCKET=publishing"
- "DOCS_BUCKET=docs"
- "NPM_BUCKET=npm"
- "REGISTRY_URL=http://localhost:4873"
- "NPM_URL=http://example.com/unused"
jsr:
depends_on:
jsr-api:
condition: service_healthy
image: nginx:1.21
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf
ports:
- "4873:80"
# our stuff
build:
build:
context: ../..
dockerfile: e2e/deno/Dockerfile.build
environment:
- GITHUB_TOKEN=${GITHUB_TOKEN}
depends_on:
- jsr
test:
build:
context: .
dockerfile: Dockerfile.test
environment:
- API_ID=${API_ID}
- API_HASH=${API_HASH}
depends_on:
- jsr
networks:
mtcute-e2e: {}

67
e2e/deno/init-server.js Normal file
View file

@ -0,0 +1,67 @@
/* eslint-disable no-console */
const { execSync } = require('child_process')
function getDockerContainerIp(name) {
const containerId = execSync(`docker compose ps -q ${name}`).toString().trim()
const ip = execSync(`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${containerId}`)
.toString()
.trim()
return ip
}
for (const stmt of [
"delete from tokens where user_id = '00000000-0000-0000-0000-000000000000';",
"insert into tokens (hash, user_id, type, expires_at) values ('3c469e9d6c5875d37a43f353d4f88e61fcf812c66eee3457465a40b0da4153e0', '00000000-0000-0000-0000-000000000000', 'web', current_date + interval '100' year);",
"update users set is_staff = true, scope_limit = 99999 where id = '00000000-0000-0000-0000-000000000000';",
]) {
execSync(`docker compose exec jsr-db psql registry -U user -c "${stmt}"`)
}
console.log('[i] Initialized database')
const GCS_URL = `http://${getDockerContainerIp('jsr-gcs')}:4080/`
const API_URL = `http://${getDockerContainerIp('jsr-api')}:8001/`
async function createBucket(name) {
try {
const resp = await fetch(`${GCS_URL}storage/v1/b`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name }),
})
await resp.text()
return resp.ok || resp.status === 409
} catch (e) {
console.log(e)
return false
}
}
(async () => {
for (const bucket of ['modules', 'docs', 'publishing', 'npm']) {
const ok = await createBucket(bucket)
console.log(`[i] Created bucket ${bucket}: ${ok}`)
}
// create @mtcute scope if it doesn't exist
const resp = await fetch(`${API_URL}api/scopes`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Cookie: 'token=token',
},
body: JSON.stringify({ scope: 'mtcute' }),
})
if (resp.status !== 200 && resp.status !== 409) {
throw new Error(`Failed to create scope: ${resp.statusText} ${await resp.text()}`)
}
if (resp.status === 200) {
console.log('[i] Created scope mtcute')
}
})()

29
e2e/deno/nginx.conf Normal file
View file

@ -0,0 +1,29 @@
events {}
http {
upstream gcs {
server jsr-gcs:4080;
}
upstream api {
server jsr-api:8001;
}
error_log /error.log debug;
server {
listen 80;
location ~ ^/(@.*)$ {
proxy_pass http://gcs/storage/v1/b/modules/o/$1?alt=media;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
proxy_pass http://api;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
}

View file

@ -0,0 +1,21 @@
import { assertEquals } from 'https://deno.land/std@0.223.0/assert/mod.ts'
import { BaseTelegramClient } from '@mtcute/core/client.js'
import { getApiParams } from '../../utils.ts'
Deno.test('@mtcute/core', async (t) => {
await t.step('connects to test DC and makes help.getNearestDc', async () => {
const tg = new BaseTelegramClient({
...getApiParams(),
})
await tg.connect()
const config = await tg.call({ _: 'help.getNearestDc' })
await tg.close()
assertEquals(typeof config, 'object')
assertEquals(config._, 'nearestDc')
assertEquals(config.thisDc, 2)
})
})

View file

@ -0,0 +1,73 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { assertEquals } from 'https://deno.land/std@0.223.0/assert/mod.ts'
import { Long } from '@mtcute/core'
import { setPlatform } from '@mtcute/core/platform.js'
import { TlBinaryReader, TlBinaryWriter, TlSerializationCounter } from '@mtcute/tl-runtime'
import { WebPlatform } from '@mtcute/web'
// here we primarily want to check that everything imports properly,
// and that the code is actually executable. The actual correctness
// of the implementation is covered tested by unit tests
const p = new WebPlatform()
setPlatform(p)
Deno.test('encodings', () => {
assertEquals(p.hexEncode(new Uint8Array([1, 2, 3, 4, 5])), '0102030405')
})
Deno.test('TlBinaryReader', () => {
const map = {
'85337187': function (r: any) {
const ret: any = {}
ret._ = 'mt_resPQ'
ret.nonce = r.int128()
ret.serverNonce = r.int128()
ret.pq = r.bytes()
ret.serverPublicKeyFingerprints = r.vector(r.long)
return ret
},
}
const data =
'000000000000000001c8831ec97ae55140000000632416053e0549828cca27e966b301a48fece2fca5cf4d33f4a11ea877ba4aa5739073300817ed48941a08f98100000015c4b51c01000000216be86c022bb4c3'
const buf = p.hexDecode(data)
const r = new TlBinaryReader(map, buf, 8)
assertEquals(r.long().toString(16), '51e57ac91e83c801')
assertEquals(r.uint(), 64)
const obj: any = r.object()
assertEquals(obj._, 'mt_resPQ')
})
Deno.test('TlBinaryWriter', () => {
const map = {
mt_resPQ: function (w: any, obj: any) {
w.uint(85337187)
w.bytes(obj.pq)
w.vector(w.long, obj.serverPublicKeyFingerprints)
},
_staticSize: {} as any,
}
const obj = {
_: 'mt_resPQ',
pq: p.hexDecode('17ED48941A08F981'),
serverPublicKeyFingerprints: [Long.fromString('c3b42b026ce86b21', 16)],
}
assertEquals(TlSerializationCounter.countNeededBytes(map, obj), 32)
const w = TlBinaryWriter.alloc(map, 48)
w.long(Long.ZERO)
w.long(Long.fromString('51E57AC91E83C801', true, 16)) // messageId
w.object(obj)
assertEquals(
p.hexEncode(w.result()),
'000000000000000001c8831ec97ae551632416050817ed48941a08f98100000015c4b51c01000000216be86c022bb4c3',
)
})

View file

@ -0,0 +1,43 @@
import { assertEquals } from 'https://deno.land/std@0.223.0/assert/mod.ts'
import { Long } from '@mtcute/core'
import { setPlatform } from '@mtcute/core/platform.js'
import { tl } from '@mtcute/tl'
import { __tlReaderMap } from '@mtcute/tl/binary/reader.js'
import { __tlWriterMap } from '@mtcute/tl/binary/writer.js'
import { TlBinaryReader, TlBinaryWriter } from '@mtcute/tl-runtime'
import { WebPlatform } from '@mtcute/web'
// here we primarily want to check that @mtcute/tl correctly works with @mtcute/tl-runtime
const p = new WebPlatform()
setPlatform(p)
Deno.test('@mtcute/tl', async (t) => {
await t.step('writers map works with TlBinaryWriter', () => {
const obj = {
_: 'inputPeerUser',
userId: 123,
accessHash: Long.fromNumber(456),
}
assertEquals(
p.hexEncode(TlBinaryWriter.serializeObject(__tlWriterMap, obj)),
'4ca5e8dd7b00000000000000c801000000000000',
)
})
await t.step('readers map works with TlBinaryReader', () => {
const buf = p.hexDecode('4ca5e8dd7b00000000000000c801000000000000')
// eslint-disable-next-line
const obj = TlBinaryReader.deserializeObject<any>(__tlReaderMap, buf)
assertEquals(obj._, 'inputPeerUser')
assertEquals(obj.userId, 123)
assertEquals(obj.accessHash.toString(), '456')
})
await t.step('correctly checks for combinator types', () => {
assertEquals(tl.isAnyInputUser({ _: 'inputUserEmpty' }), true)
})
})

View file

@ -0,0 +1,25 @@
import { assertEquals } from 'https://deno.land/std@0.223.0/assert/mod.ts'
import { ige256Decrypt, ige256Encrypt } from '@mtcute/wasm'
import { WebCryptoProvider, WebPlatform } from '@mtcute/web'
await new WebCryptoProvider().initialize()
const platform = new WebPlatform()
Deno.test('@mtcute/wasm', async (t) => {
const key = platform.hexDecode('5468697320697320616E20696D706C655468697320697320616E20696D706C65')
const iv = platform.hexDecode('6D656E746174696F6E206F6620494745206D6F646520666F72204F70656E5353')
const data = platform.hexDecode('99706487a1cde613bc6de0b6f24b1c7aa448c8b9c3403e3467a8cad89340f53b')
const dataEnc = platform.hexDecode('792ea8ae577b1a66cb3bd92679b8030ca54ee631976bd3a04547fdcb4639fa69')
await t.step('should work with Buffers', () => {
assertEquals(ige256Encrypt(data, key, iv), dataEnc)
assertEquals(ige256Decrypt(dataEnc, key, iv), data)
})
await t.step('should work with Uint8Arrays', () => {
assertEquals(ige256Encrypt(data, key, iv), dataEnc)
assertEquals(ige256Decrypt(dataEnc, key, iv), data)
})
})

42
e2e/deno/utils.ts Normal file
View file

@ -0,0 +1,42 @@
import { MaybePromise, MemoryStorage } from '@mtcute/core'
import { setPlatform } from '@mtcute/core/platform.js'
import { LogManager, sleep } from '@mtcute/core/utils.js'
import { WebCryptoProvider, WebPlatform, WebSocketTransport } from '@mtcute/web'
export const getApiParams = (storage?: string) => {
if (storage) throw new Error('unsupported yet')
if (!Deno.env.has('API_ID') || !Deno.env.has('API_HASH')) {
throw new Error('API_ID and API_HASH env variables must be set')
}
setPlatform(new WebPlatform())
return {
apiId: parseInt(Deno.env.get('API_ID')!),
apiHash: Deno.env.get('API_HASH')!,
testMode: true,
storage: new MemoryStorage(),
logLevel: LogManager.VERBOSE,
transport: () => new WebSocketTransport(),
crypto: new WebCryptoProvider(),
}
}
export async function waitFor(condition: () => MaybePromise<void>, timeout = 5000): Promise<void> {
const start = Date.now()
let lastError
while (Date.now() - start < timeout) {
try {
await condition()
return
} catch (e) {
lastError = e
await sleep(100)
}
}
throw lastError
}

View file

@ -1,3 +1,5 @@
# obtain these values from my.telegram.org # obtain these values from my.telegram.org
API_ID= API_ID=
API_HASH= API_HASH=
GITHUB_TOKEN=

View file

@ -1,6 +1,10 @@
module.exports = ({ path: { join }, fs, outDir, packageDir, jsr }) => ({ module.exports = ({ path: { join }, fs, outDir, packageDir, jsr, transformFile }) => ({
esmOnlyDirectives: true, esmOnlyDirectives: true,
final() { final() {
fs.cpSync(join(packageDir, 'mtcute.wasm'), join(outDir, 'mtcute.wasm')) fs.cpSync(join(packageDir, 'mtcute.wasm'), join(outDir, 'mtcute.wasm'))
if (jsr) {
transformFile(join(outDir, 'index.ts'), (code) => code.replace("'../mtcute.wasm'", "'./mtcute.wasm'"))
}
}, },
}) })

View file

@ -4,8 +4,17 @@ const cp = require('child_process')
const IS_JSR = process.env.JSR === '1' const IS_JSR = process.env.JSR === '1'
const MAIN_REGISTRY = IS_JSR ? 'http://jsr.test/' : 'https://registry.npmjs.org' const MAIN_REGISTRY = IS_JSR ? 'http://jsr.test/' : 'https://registry.npmjs.org'
const REGISTRY = process.env.REGISTRY || MAIN_REGISTRY let REGISTRY = process.env.REGISTRY || MAIN_REGISTRY
exports.REGISTRY = REGISTRY exports.REGISTRY = REGISTRY
if (!REGISTRY.endsWith('/')) REGISTRY += '/'
if (process.env.E2E && IS_JSR) {
// running behind a socat proxy seems to fix some of the docker networking issues (thx kamillaova)
const hostname = new URL(REGISTRY).hostname
const port = new URL(REGISTRY).port || { 'http:': 80, 'https:': 443 }[new URL(REGISTRY).protocol]
cp.spawn('bash', ['-c', `socat TCP-LISTEN:1234,fork,reuseaddr TCP4:${hostname}:${port}`], { stdio: 'ignore' })
REGISTRY = 'http://localhost:1234/'
}
if (IS_JSR) { if (IS_JSR) {
// for the underlying tools that expect JSR_URL env var // for the underlying tools that expect JSR_URL env var
@ -23,26 +32,49 @@ const JSR_EXCEPTIONS = {
test: 'never', test: 'never',
} }
async function checkVersion(name, version, retry = 0) { function fetchRetry(url, init, retry = 0) {
return fetch(url, init).catch((err) => {
if (retry >= 5) throw err
// for whatever reason this request sometimes fails with ECONNRESET
// no idea why, probably some issue in docker networking
console.log('[i] Error fetching %s:', url)
console.log(err)
return new Promise((resolve) => setTimeout(resolve, 1000)).then(() => fetchRetry(url, init, retry + 1))
})
}
async function checkVersion(name, version) {
let registry = REGISTRY let registry = REGISTRY
if (!registry.endsWith('/')) registry += '/'
const url = IS_JSR ? `${registry}@mtcute/${name}/${version}_meta.json` : `${registry}@mtcute/${name}/${version}` const url = IS_JSR ? `${registry}@mtcute/${name}/${version}_meta.json` : `${registry}@mtcute/${name}/${version}`
return fetch(url) return fetchRetry(url).then((r) => r.status === 200)
.then((r) => r.status === 200) }
.catch((err) => {
if (retry >= 5) throw err
// for whatever reason this request sometimes fails with ECONNRESET async function jsrMaybeCreatePackage(name) {
// no idea why, probably some issue in docker networking // check if the package even exists
console.log('[i] Error checking version:') const packageMeta = await fetchRetry(`${REGISTRY}api/scopes/mtcute/packages/${name}`)
console.log(err)
return new Promise((resolve) => setTimeout(resolve, 1000)).then(() => if (packageMeta.status === 404) {
checkVersion(name, version, retry + 1), console.error('[i] %s does not exist, creating..', name)
)
const create = await fetchRetry(`${REGISTRY}api/scopes/mtcute/packages`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Cookie: `token=${process.env.JSR_TOKEN}`,
},
body: JSON.stringify({ package: name }),
}) })
if (create.status !== 200) {
throw new Error(`Failed to create package: ${create.statusText} ${await create.text()}`)
}
} else if (packageMeta.status !== 200) {
throw new Error(`Failed to check package: ${packageMeta.statusText} ${await packageMeta.text()}`)
}
} }
async function publishSinglePackage(name) { async function publishSinglePackage(name) {
@ -76,11 +108,14 @@ async function publishSinglePackage(name) {
return return
} }
} else if (IS_JSR && process.env.JSR_TOKEN) {
await jsrMaybeCreatePackage(name)
} }
if (IS_JSR) { if (IS_JSR) {
// publish to jsr // publish to jsr
cp.execSync('deno publish --allow-dirty', { const params = process.env.JSR_TOKEN ? `--token ${process.env.JSR_TOKEN}` : ''
cp.execSync(`deno publish --allow-dirty ${params}`, {
cwd: path.join(packageDir, 'dist/jsr'), cwd: path.join(packageDir, 'dist/jsr'),
stdio: 'inherit', stdio: 'inherit',
}) })
@ -158,6 +193,7 @@ async function main(arg = process.argv[2]) {
} catch (e) { } catch (e) {
console.error('[!] Failed to publish %s:', pkg) console.error('[!] Failed to publish %s:', pkg)
console.error(e) console.error(e)
if (IS_JSR || process.env.E2E) throw e
failedPkgs.push(pkg) failedPkgs.push(pkg)
} }
} }
@ -169,6 +205,8 @@ async function main(arg = process.argv[2]) {
} catch (e) { } catch (e) {
console.error('[!] Failed to publish %s:', pkg) console.error('[!] Failed to publish %s:', pkg)
console.error(e) console.error(e)
if (IS_JSR || process.env.E2E) throw e
failedPkgs.push(pkg) failedPkgs.push(pkg)
} }
} }