mirror of
https://codeberg.org/shroff/phylum.git
synced 2026-01-05 19:21:23 -06:00
[server] Use river queue for backgroud jobs
This commit is contained in:
Abhishek Shroff
@@ -12,7 +12,7 @@ require (
|
||||
github.com/gin-contrib/static v1.1.5
|
||||
github.com/gin-gonic/gin v1.10.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.5.5
|
||||
github.com/jackc/pgx/v5 v5.7.5
|
||||
github.com/jackc/tern/v2 v2.2.1
|
||||
github.com/knadh/koanf/parsers/yaml v1.0.0
|
||||
github.com/knadh/koanf/providers/file v1.2.0
|
||||
@@ -20,15 +20,28 @@ require (
|
||||
github.com/knadh/koanf/providers/rawbytes v1.0.0
|
||||
github.com/knadh/koanf/v2 v2.2.0
|
||||
github.com/minio/minio-go/v7 v7.0.69
|
||||
github.com/riverqueue/river v0.23.1
|
||||
github.com/riverqueue/river/riverdriver/riverpgxv5 v0.23.1
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/spf13/cobra v1.8.0
|
||||
github.com/spf13/pflag v1.0.6
|
||||
golang.org/x/crypto v0.37.0
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0
|
||||
golang.org/x/term v0.31.0
|
||||
)
|
||||
|
||||
require gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
require (
|
||||
github.com/riverqueue/river/riverdriver v0.23.1 // indirect
|
||||
github.com/riverqueue/river/rivershared v0.23.1 // indirect
|
||||
github.com/riverqueue/river/rivertype v0.23.1 // indirect
|
||||
github.com/stretchr/testify v1.10.0 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
go.uber.org/goleak v1.3.0 // indirect
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||
@@ -50,8 +63,8 @@ require (
|
||||
github.com/imdario/mergo v0.3.16 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.17.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
@@ -74,9 +87,9 @@ require (
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
golang.org/x/arch v0.16.0 // indirect
|
||||
golang.org/x/net v0.38.0 // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sync v0.14.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
golang.org/x/text v0.25.0 // indirect
|
||||
google.golang.org/protobuf v1.36.6 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/mail.v2 v2.3.1
|
||||
|
||||
@@ -55,8 +55,8 @@ github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -69,14 +69,16 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
|
||||
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0=
|
||||
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
|
||||
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
|
||||
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
|
||||
github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jackc/tern/v2 v2.2.1 h1:kricKrvA6FNzBHHaQu15hmJDnpHvZA2DoJa97lJLt10=
|
||||
github.com/jackc/tern/v2 v2.2.1/go.mod h1:thNyC7gVBGYWsAJJSvAX0ML/1lAmOw7+DVH8aSE5rto=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
@@ -108,8 +110,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
|
||||
github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
@@ -130,13 +133,33 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
|
||||
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/riverqueue/river v0.23.1 h1:/iwpDJ4ypgoVNMDDtQ7PYUKQd+lk6z414fGmp3nei84=
|
||||
github.com/riverqueue/river v0.23.1/go.mod h1:+02PXpjXtHnV5QzARe9BfltC52Kcm8y+BzaD6s6a2J4=
|
||||
github.com/riverqueue/river/riverdriver v0.23.1 h1:KG7uUg2l2TWsPGcDfYD3U2ZAHXnZ/iZNH+JT0LjOq20=
|
||||
github.com/riverqueue/river/riverdriver v0.23.1/go.mod h1:GN3r8XgDN/YwY1mudkPdrtyFTE3Pq/AMKrUePlcH0Uc=
|
||||
github.com/riverqueue/river/riverdriver/riverdatabasesql v0.23.1 h1:WIVKfmyprocrZfSjtM5lNNu+Hul+r64HHoR1CEbQ1g0=
|
||||
github.com/riverqueue/river/riverdriver/riverdatabasesql v0.23.1/go.mod h1:v9OaTsxzr52ZCjGdfsaV5OIIQL84fcFuENQzaVRV5gI=
|
||||
github.com/riverqueue/river/riverdriver/riverpgxv5 v0.23.1 h1:hztWRKCHcsf9jkSjBCfQ6FQgoKoCtmd6A8EualE4ZEk=
|
||||
github.com/riverqueue/river/riverdriver/riverpgxv5 v0.23.1/go.mod h1:Wn8rY1a3a4I5nvskpebNK+LCkkopVFTUNPW9UklW02g=
|
||||
github.com/riverqueue/river/riverdriver/riversqlite v0.23.1 h1:v5onNmGdbsmyQAIYG3I77/RG8wkfrAOrLRTAJ8vXgNU=
|
||||
github.com/riverqueue/river/riverdriver/riversqlite v0.23.1/go.mod h1:yRc5N+kod5r4oIvHSK9GNDddP13zm1/VEFwG55pYhO8=
|
||||
github.com/riverqueue/river/rivershared v0.23.1 h1:ZC6ybv5KguD/mpLkaXrtUCES6FyKbGsavk25YNJdp0s=
|
||||
github.com/riverqueue/river/rivershared v0.23.1/go.mod h1:8/jFVQNfUesv5y+qQZ55XULMCOdM5yj9F4MG7/UA8LA=
|
||||
github.com/riverqueue/river/rivertype v0.23.1 h1:vaIIm54BVzvy2iXT/iP7isIPSv2k99DElJNI6hWQ1lc=
|
||||
github.com/riverqueue/river/rivertype v0.23.1/go.mod h1:lmdl3vLNDfchDWbYdW2uAocIuwIN+ZaXqAukdSCFqWs=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
@@ -168,11 +191,23 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U=
|
||||
golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
@@ -182,8 +217,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@@ -194,8 +229,8 @@ golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
||||
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -216,8 +251,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
||||
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
@@ -238,4 +273,12 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/libc v1.64.0 h1:U0k8BD2d3cD3e9I8RLcZgJBHAcsJzbXx5mKGSb5pyJA=
|
||||
modernc.org/libc v1.64.0/go.mod h1:7m9VzGq7APssBTydds2zBcxGREwvIGpuUBaKTXdm2Qs=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.10.0 h1:fzumd51yQ1DxcOxSO+S6X7+QTuVU+n8/Aj7swYjFfC4=
|
||||
modernc.org/memory v1.10.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI=
|
||||
modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
apiv1 "github.com/shroff/phylum/server/internal/api/v1"
|
||||
"github.com/shroff/phylum/server/internal/api/webdav"
|
||||
"github.com/shroff/phylum/server/internal/core"
|
||||
"github.com/shroff/phylum/server/internal/db"
|
||||
"github.com/shroff/phylum/server/internal/jobs"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -56,6 +58,11 @@ func SetupCommand() *cobra.Command {
|
||||
publink.Setup(engine.Group(Cfg.PublinkPath))
|
||||
|
||||
setupWebApp(engine, "web")
|
||||
|
||||
err := jobs.Initialize(context.Background(), db.Pool())
|
||||
if err != nil {
|
||||
logrus.Fatal("Failed to initializat jobs: " + err.Error())
|
||||
}
|
||||
setupTrashCompactor()
|
||||
|
||||
listen := Cfg.Host + ":" + strconv.Itoa(Cfg.Port)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/shroff/phylum/server/internal/jobs"
|
||||
)
|
||||
|
||||
func (f filesystem) deleteChildRecursive(r Resource, name string, softDelete bool) (Resource, error) {
|
||||
@@ -175,7 +176,7 @@ func (f filesystem) deleteRecursive(id, parent uuid.UUID, softDelete, preserveRo
|
||||
})
|
||||
|
||||
if err == nil && !softDelete {
|
||||
go deleteAllVersionContents(f.db, ids)
|
||||
jobs.DeleteAllVersionContents(ids)
|
||||
}
|
||||
|
||||
return err
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"hash"
|
||||
"io"
|
||||
@@ -9,7 +8,7 @@ import (
|
||||
"github.com/doug-martin/goqu/v9"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/shroff/phylum/server/internal/db"
|
||||
"github.com/shroff/phylum/server/internal/jobs"
|
||||
"github.com/shroff/phylum/server/internal/storage"
|
||||
)
|
||||
|
||||
@@ -41,7 +40,7 @@ func (f filesystem) OpenWrite(r Resource, versionID uuid.UUID) (io.WriteCloser,
|
||||
return err
|
||||
}
|
||||
|
||||
go migrateVersionContents(db.Get(context.Background()), versionID)
|
||||
jobs.MigrateVersionContents(versionID)
|
||||
return nil
|
||||
}), nil
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/shroff/phylum/server/internal/db"
|
||||
"github.com/shroff/phylum/server/internal/jobs"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -108,7 +109,7 @@ func (f filesystem) TrashEmpty() (int, error) {
|
||||
Returning("id", "dir")
|
||||
query, args, _ := q.ToSQL()
|
||||
count, ids, err := collectNonDirResourceIDs(f.db.Query(query, args...))
|
||||
deleteAllVersionContents(f.db, ids)
|
||||
jobs.DeleteAllVersionContents(ids)
|
||||
return count, err
|
||||
}
|
||||
|
||||
@@ -134,6 +135,6 @@ func hardDeleteOldResources(ctx context.Context, t time.Time) (int, error) {
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
deleteAllVersionContents(db, ids)
|
||||
jobs.DeleteAllVersionContents(ids)
|
||||
return count, err
|
||||
}
|
||||
|
||||
@@ -33,6 +33,12 @@ func Get(ctx context.Context) Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// Pool returns the pgx pool
|
||||
// Only to be used for jobs initialization
|
||||
func Pool() *pgxpool.Pool {
|
||||
return pool
|
||||
}
|
||||
|
||||
func (d Handler) Notifier() pubsub.Notifier {
|
||||
return notifier
|
||||
}
|
||||
|
||||
@@ -1,17 +1,38 @@
|
||||
package core
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/riverqueue/river"
|
||||
"github.com/shroff/phylum/server/internal/db"
|
||||
"github.com/shroff/phylum/server/internal/storage"
|
||||
)
|
||||
|
||||
type DeleteArgs struct {
|
||||
VersionIDS uuid.UUIDs `json:"version_ids"`
|
||||
}
|
||||
|
||||
func (DeleteArgs) Kind() string { return "delete" }
|
||||
|
||||
type DeleteWorker struct {
|
||||
river.WorkerDefaults[DeleteArgs]
|
||||
}
|
||||
|
||||
func (w *DeleteWorker) Work(ctx context.Context, job *river.Job[DeleteArgs]) error {
|
||||
return deleteAllVersionContents(db.Get(ctx), job.Args.VersionIDS)
|
||||
}
|
||||
|
||||
type versionInfo struct {
|
||||
id uuid.UUID
|
||||
storage string
|
||||
}
|
||||
|
||||
func DeleteAllVersionContents(ids uuid.UUIDs) {
|
||||
client.Insert(context.Background(), DeleteArgs{VersionIDS: ids}, &river.InsertOpts{})
|
||||
}
|
||||
|
||||
func deleteAllVersionContents(db db.Handler, ids uuid.UUIDs) error {
|
||||
const q = `SELECT v.id, v.storage FROM resources r
|
||||
JOIN resource_versions v ON r.id = v.resource_id
|
||||
61
server/internal/jobs/jobs.go
Normal file
61
server/internal/jobs/jobs.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/riverqueue/river"
|
||||
"github.com/riverqueue/river/riverdriver/riverpgxv5"
|
||||
"github.com/riverqueue/river/rivermigrate"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// var client river.Client
|
||||
|
||||
var client *river.Client[pgx.Tx]
|
||||
|
||||
func Initialize(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
logrus.Info("Initializing Jobs")
|
||||
if client != nil {
|
||||
return errors.New("jobs already initialized")
|
||||
}
|
||||
|
||||
if m, err := rivermigrate.New(riverpgxv5.New(pool), nil); err != nil {
|
||||
return err
|
||||
} else if _, err := m.Migrate(ctx, rivermigrate.DirectionUp, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
workers := river.NewWorkers()
|
||||
river.AddWorker(workers, &MigrateWorker{})
|
||||
river.AddWorker(workers, &DeleteWorker{})
|
||||
|
||||
if c, err := river.NewClient(riverpgxv5.New(pool), &river.Config{
|
||||
Queues: map[string]river.QueueConfig{
|
||||
river.QueueDefault: {MaxWorkers: 5},
|
||||
},
|
||||
Workers: workers,
|
||||
}); err != nil {
|
||||
return err
|
||||
} else {
|
||||
client = c
|
||||
}
|
||||
return client.Start(ctx)
|
||||
}
|
||||
|
||||
func Shutdown(ctx context.Context) {
|
||||
logrus.Info("Stopping Jobs")
|
||||
gracefulCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := client.Stop(gracefulCtx); err != nil {
|
||||
logrus.Info("No more Mr. Nice Guy")
|
||||
hardStopCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
if err := client.StopAndCancel(hardStopCtx); err != nil {
|
||||
logrus.Warn("Could not stop and cancel jobs: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,35 @@
|
||||
package core
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/riverqueue/river"
|
||||
"github.com/shroff/phylum/server/internal/db"
|
||||
"github.com/shroff/phylum/server/internal/storage"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func migrateVersionContents(db db.Handler, versionID uuid.UUID) {
|
||||
if err := moveVersionToTargetBackendErr(db, versionID); err != nil {
|
||||
logrus.Warn("failed to move " + versionID.String() + " to target storage: " + err.Error())
|
||||
}
|
||||
type MigrateArgs struct {
|
||||
VersionID uuid.UUID `json:"version_id"`
|
||||
}
|
||||
func moveVersionToTargetBackendErr(db db.Handler, versionID uuid.UUID) error {
|
||||
|
||||
func (MigrateArgs) Kind() string { return "migrate" }
|
||||
|
||||
type MigrateWorker struct {
|
||||
river.WorkerDefaults[MigrateArgs]
|
||||
}
|
||||
|
||||
func (w *MigrateWorker) Work(ctx context.Context, job *river.Job[MigrateArgs]) error {
|
||||
return migrateVersionContents(db.Get(ctx), job.Args.VersionID)
|
||||
}
|
||||
|
||||
func MigrateVersionContents(versionID uuid.UUID) {
|
||||
client.Insert(context.Background(), MigrateArgs{VersionID: versionID}, &river.InsertOpts{})
|
||||
}
|
||||
|
||||
func migrateVersionContents(db db.Handler, versionID uuid.UUID) error {
|
||||
if current, err := getCurrentStorage(db, versionID); err != nil {
|
||||
return errors.New("failed to get current storage: " + err.Error())
|
||||
} else if target, err := findTargetStorageForVersion(db, versionID); err != nil {
|
||||
Reference in New Issue
Block a user