commit f1dfd5bfb4ff9ad745980ee11232dbeb54ba3633
parent 047be2f0eb8b5160dc8984f1b9492074a859d3bb
Author: mtmn <miro@haravara.org>
Date: Sat, 11 Apr 2026 07:48:51 +0200
chore: cleanup, builds
Diffstat:
| M | .gitignore | | | 1 | - |
| M | Containerfile | | | 19 | +++++++++++++++---- |
| M | flake.nix | | | 67 | ++++++++++++++++++------------------------------------------------- |
| M | src/Main.purs | | | 86 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------ |
4 files changed, 93 insertions(+), 80 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -13,7 +13,6 @@ result-*
.spago/
.env
.env.local
-
server.js
client.js
server.log
diff --git a/Containerfile b/Containerfile
@@ -1,4 +1,4 @@
-# docker.io/node:25-slim
+# docker.io/node:24-slim
FROM docker.io/library/node@sha256:435f3537a088a01fd208bb629a4b69c28d85deb9a60af8a710cafc3befd6e3be as builder
WORKDIR /app
@@ -7,21 +7,32 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
git \
libtinfo5 \
+ ca-certificates \
+ python3 \
+ make \
+ g++ \
&& rm -rf /var/lib/apt/lists/*
COPY package.json package-lock.json ./
COPY spago.yaml spago.lock ./
-
-RUN npm ci
COPY src ./src
-RUN npm run build
+RUN npm ci && \
+ npx spago build --jobs 8 && \
+ npx esbuild output/Main/index.js --bundle --platform=node --format=esm --outfile=server.js --external:http --external:https --external:dotenv --external:url --external:duckdb --external:@aws-sdk/client-s3 && \
+ npx spago bundle --module Client --outfile client.js --platform browser
# docker.io/node:25-slim
FROM docker.io/library/node@sha256:435f3537a088a01fd208bb629a4b69c28d85deb9a60af8a710cafc3befd6e3be as builder
WORKDIR /app
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ python3 \
+ make \
+ g++ \
+ && rm -rf /var/lib/apt/lists/*
+
COPY package.json package-lock.json ./
RUN npm ci --omit=dev
diff --git a/flake.nix b/flake.nix
@@ -13,33 +13,36 @@
flake-utils.lib.eachDefaultSystem (
system: let
pkgs = nixpkgs.legacyPackages.${system};
-
- scorpus = pkgs.buildNpmPackage {
+ in {
+ packages.default = pkgs.buildNpmPackage {
pname = "scorpus";
version = "1.0.0";
- src = self;
+ src = pkgs.lib.cleanSource ./.;
npmDepsHash = "sha256-L+9L3RY5CNLudmQL7EZwZO8x0kQ2MKCqSsn37ZIIKWE=";
- nativeBuildInputs = with pkgs; [
- purescript
- esbuild
- makeWrapper
- python3
- pkg-config
- ];
+ nativeBuildInputs = with pkgs; [ purescript spago esbuild makeWrapper ];
- npmFlags = ["--ignore-scripts"];
+ # npm install will try to download purescript/spago binaries by default.
+ # We want to use the ones from nixpkgs.
+ PURESCRIPT_DOWNLOAD_BINARY = "0";
+ SPAGO_DOWNLOAD_BINARY = "0";
+ # We need to run spago build and then esbuild.
+ # buildNpmPackage's buildPhase runs 'npm run build' by default.
+ # We can override it if we want.
buildPhase = ''
export HOME=$TMPDIR
- export PATH=$PWD/node_modules/.bin:$PATH
- spago build && esbuild output/Main/index.js --bundle --platform=node --format=esm --outfile=server.js --external:http --external:https --external:dotenv --external:url --external:duckdb --external:@aws-sdk/client-s3 && spago bundle --module Client --outfile client.js --platform browser
+ # Since we are using pkgs.spago, we might need to tell it where to find packages.
+ # But it will still try to download them if they are not there.
+ # For now, let's see what happens.
+ npm run build
'';
installPhase = ''
mkdir -p $out/lib/scorpus
- cp -r server.js client.js package.json node_modules $out/lib/scorpus/
+ cp server.js client.js package.json $out/lib/scorpus/
+ cp -r node_modules $out/lib/scorpus/
mkdir -p $out/bin
makeWrapper ${pkgs.nodejs}/bin/node $out/bin/scorpus-server \
@@ -47,43 +50,9 @@
--add-flags "$out/lib/scorpus/server.js"
'';
};
- in {
- packages.default = scorpus;
-
- packages.oci = pkgs.dockerTools.buildLayeredImage {
- name = "scorpus";
- tag = "latest";
- contents = [pkgs.nodejs pkgs.cacert];
- config = {
- Cmd = ["${scorpus}/bin/scorpus-server"];
- WorkingDir = "/app";
- ExposedPorts = {
- "8321/tcp" = {};
- };
- User = "1000:1000";
- Env = [
- "PORT=8321"
- "DATABASE_FILE=/app/data/scorpus.db"
- "NODE_ENV=production"
- ];
- };
- extraCommands = ''
- mkdir -p app/data
- chown -R 1000:1000 app
- chmod 700 app/data
- '';
- };
devShells.default = pkgs.mkShell {
- buildInputs = with pkgs; [
- nodejs
- purescript
- awscli2
- duckdb
- spago
- purs-tidy
- esbuild
- ];
+ buildInputs = with pkgs; [ nodejs esbuild purs-tidy ];
};
}
);
diff --git a/src/Main.purs b/src/Main.purs
@@ -36,6 +36,7 @@ import Data.Argonaut (decodeJson, encodeJson, parseJson)
import Data.Argonaut.Core (toObject, toArray, toString, stringify)
import Data.Array ((!!), length, uncons)
import Data.Nullable (Nullable, toMaybe)
+import Data.Tuple (Tuple(..))
import Db (Connection, connect, initDb, upsertScrobble, getScrobbles, checkExists, run)
import Types (Listen(..), ListenBrainzResponse(..), Payload(..), TrackMetadata(..))
import Control.Monad.Rec.Class (forever)
@@ -53,12 +54,9 @@ listenBrainzUrl = "https://api.listenbrainz.org/1/user/mtmn/listens"
fetchListenBrainzData :: Int -> Aff String
fetchListenBrainzData count = makeAff \callback -> do
let url = listenBrainzUrl <> "?count=" <> show count
- Log.info $ "Fetching from ListenBrainz: " <> url
req <- HTTPS.get url
req # on_ Client.responseH \res -> do
- let sc = IM.statusCode res
- Log.info $ "ListenBrainz response status: " <> show sc
launchAff_ do
body <- readableToStringUtf8 (IM.toReadable res)
liftEffect $ callback (Right body)
@@ -73,51 +71,87 @@ fetchListenBrainzData count = makeAff \callback -> do
pure nonCanceler
-syncData :: Connection -> Aff Unit
-syncData conn = do
- env <- liftEffect getEnv
- let shouldInitialSync = Object.lookup "INITIAL_SYNC" env == Just "true"
+fetchListenBrainzDataBefore :: Int -> Aff String
+fetchListenBrainzDataBefore maxTs = makeAff \callback -> do
+ let url = listenBrainzUrl <> "?count=100&max_ts=" <> show maxTs
+ req <- HTTPS.get url
+
+ req # on_ Client.responseH \res -> do
+ launchAff_ do
+ body <- readableToStringUtf8 (IM.toReadable res)
+ liftEffect $ callback (Right body)
+
+ let errorH = EventHandle "error" mkEffectFn1
+ on_ errorH
+ ( \err -> do
+ Log.error $ "ListenBrainz fetch error: " <> Exception.message err
+ callback (Left err)
+ )
+ (unsafeCoerce req)
- if shouldInitialSync then do
- Log.info "Performing initial sync of 100 tracks"
- void $ performSync 100
- else
- Log.info "Initial sync skipped (INITIAL_SYNC != true)"
+ pure nonCanceler
+syncData :: Connection -> Aff Unit
+syncData conn = do
forever do
- Log.info "Performing periodic sync..."
- void $ performSync 100
+ void $ performFullSync
delay (Milliseconds 60000.0)
where
- performSync count = do
- result <- try $ fetchListenBrainzData count
+ performFullSync = do
+ result <- try $ fetchListenBrainzData 100
case result of
Right body -> do
case parseJson body >>= decodeJson of
Left err -> Log.error $ "Sync parse error: " <> show err
Right (ListenBrainzResponse { payload: Payload { listens } }) -> do
- Log.info $ "Processing " <> show (length listens) <> " scrobbles"
run conn "BEGIN TRANSACTION" []
- newCount <- syncRecursive 0 listens
+ Tuple added (Tuple minTs hitExisting) <- processListens listens
run conn "COMMIT" []
- Log.info $ "Sync complete. Added " <> show newCount <> " new scrobbles."
+ if hitExisting || length listens == 0 then do
+ when (added > 0) $ Log.info $ "Sync complete. Added " <> show added <> " new scrobbles."
+ else do
+ total <- paginateUntilDone minTs added
+ Log.info $ "Sync complete. Added " <> show total <> " new scrobbles."
Left err -> Log.error $ "Sync fetch error: " <> Exception.message err
- syncRecursive acc listens = case uncons listens of
+ paginateUntilDone minTs acc = case minTs of
Nothing -> pure acc
- Just { head: l@(Listen { listenedAt: Just ts, trackMetadata: (TrackMetadata track) }), tail } -> do
+ Just ts -> do
+ result <- try $ fetchListenBrainzDataBefore ts
+ case result of
+ Right body -> do
+ case parseJson body >>= decodeJson of
+ Left err -> do
+ Log.error $ "Sync parse error: " <> show err
+ pure acc
+ Right (ListenBrainzResponse { payload: Payload { listens } }) -> do
+ run conn "BEGIN TRANSACTION" []
+ Tuple added (Tuple newMinTs hitExisting) <- processListens listens
+ run conn "COMMIT" []
+ if hitExisting || length listens == 0 then do
+ pure (acc + added)
+ else do
+ paginateUntilDone newMinTs (acc + added)
+ Left err -> do
+ Log.error $ "Sync fetch error: " <> Exception.message err
+ pure acc
+
+ processListens listens = do
+ syncRecursive 0 Nothing listens
+
+ syncRecursive acc minTs listens = case uncons listens of
+ Nothing -> pure $ Tuple acc (Tuple minTs false)
+ Just { head: l@(Listen { listenedAt: Just ts, trackMetadata: (TrackMetadata _) }), tail } -> do
exists <- checkExists conn ts
if exists then do
- Log.info $ "Hit existing record at " <> show ts <> ". Stopping sync."
- pure acc
+ pure $ Tuple acc (Tuple minTs true)
else do
- Log.info $ "Adding scrobble: " <> fromMaybe "Unknown Track" track.trackName <> " - " <> fromMaybe "Unknown Artist" track.artistName <> " (" <> show ts <> ")"
upsertScrobble conn l
- syncRecursive (acc + 1) tail
+ syncRecursive (acc + 1) (Just ts) tail
Just { head: _, tail } -> do
Log.warn "Skipping scrobble without timestamp"
- syncRecursive acc tail
+ syncRecursive acc minTs tail
indexHtml :: String
indexHtml =
