[ Dec 24 05:42:44 Enabled. ]
[ Dec 24 05:42:44 Rereading configuration. ]
[ Dec 24 05:42:44 Rereading configuration. ]
[ Dec 24 05:42:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a9f9665f-3627-47fd-961a-c855e87ea460/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:42:48.363077575Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781}
{"msg":"dataset: \"oxp_a9f9665f-3627-47fd-961a-c855e87ea460/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:42:48.363853207Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:42:48.363865367Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:42:48.363872657Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:42:48.379011075Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:42:48.448741178Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.594685899Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38053"}
{"msg":"region 7a075456-3024-495f-aee3-4c53137152c8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.619930276Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.623196396Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.623249327Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"uri":"/crucible/0/regions","method":"POST","req_id":"f7d79d6e-ca19-4b62-bd4b-ead47d15e1bd","remote_addr":"[fd00:1122:3344:101::b]:38053","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14751,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_a9f9665f-3627-47fd-961a-c855e87ea460/crucible/regions/7a075456-3024-495f-aee3-4c53137152c8","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.631621118Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a9f9665f-3627-47fd-961a-c855e87ea460/crucible/regions/7a075456-3024-495f-aee3-4c53137152c8","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.631636129Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7a075456-3024-495f-aee3-4c53137152c8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7a075456-3024-495f-aee3-4c53137152c8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.704857168Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"region":"7a075456-3024-495f-aee3-4c53137152c8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7a075456-3024-495f-aee3-4c53137152c8\" \"--data\" \"/data/regions/7a075456-3024-495f-aee3-4c53137152c8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.706953595Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"region":"7a075456-3024-495f-aee3-4c53137152c8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:20.793247175Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"uri":"/crucible/0/regions","method":"POST","req_id":"dc3ba366-adce-490f-a166-b1d27d9722aa","remote_addr":"[fd00:1122:3344:101::b]:38053","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.065936615Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"region":"7a075456-3024-495f-aee3-4c53137152c8","component":"worker"}
{"msg":"region 7a075456-3024-495f-aee3-4c53137152c8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.066007677Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.066251834Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7a075456-3024-495f-aee3-4c53137152c8","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.066648385Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7a075456-3024-495f-aee3-4c53137152c8","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.071834398Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.085174336Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7a075456-3024-495f-aee3-4c53137152c8","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.089522765Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7a075456-3024-495f-aee3-4c53137152c8","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.090127342Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.090193814Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.090234915Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.090269076Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.094539994Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.099684696Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:21.406355552Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"uri":"/crucible/0/regions","method":"POST","req_id":"2aaf1256-32bd-4374-b0fa-fcda8041da83","remote_addr":"[fd00:1122:3344:101::b]:38053","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.063170115Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"uri":"/crucible/0/regions/7a075456-3024-495f-aee3-4c53137152c8","method":"GET","req_id":"363ca7a6-2be1-4676-b3c1-6a60acea3710","remote_addr":"[fd00:1122:3344:101::b]:38053","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":92,"response_code":200}
{"msg":"path is \"/data/regions/7a075456-3024-495f-aee3-4c53137152c8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.067454033Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"datafile"}
{"msg":"dataset is oxp_a9f9665f-3627-47fd-961a-c855e87ea460/crucible/regions/7a075456-3024-495f-aee3-4c53137152c8","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.075925817Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"datafile"}
{"msg":"region 7a075456-3024-495f-aee3-4c53137152c8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.08366885Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.083868516Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"uri":"/crucible/0/regions/7a075456-3024-495f-aee3-4c53137152c8","method":"DELETE","req_id":"35ea9d3b-9afe-4393-9ed1-e69144baccfa","remote_addr":"[fd00:1122:3344:101::b]:38053","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":16443,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.083919947Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-7a075456-3024-495f-aee3-4c53137152c8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.084868684Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.087739352Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"uri":"/crucible/0/regions/7a075456-3024-495f-aee3-4c53137152c8","method":"GET","req_id":"edd374ab-bf86-4849-ab59-f7704ad3f9f3","remote_addr":"[fd00:1122:3344:101::b]:38053","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":33,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.089474551Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a9f9665f-3627-47fd-961a-c855e87ea460/crucible/regions/7a075456-3024-495f-aee3-4c53137152c8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.098223742Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"region":"7a075456-3024-495f-aee3-4c53137152c8","component":"worker"}
{"msg":"region 7a075456-3024-495f-aee3-4c53137152c8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.176473399Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:43:22.212624847Z","hostname":"oxz_crucible_24bf8245-cd11-41dc-b8bf-c07a33f6cccc","pid":23781,"uri":"/crucible/0/regions/7a075456-3024-495f-aee3-4c53137152c8","method":"GET","req_id":"e0ccbb00-00ef-4e7a-985b-638b8380dd69","remote_addr":"[fd00:1122:3344:101::b]:38053","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":50,"response_code":200}