[ Nov 8 10:30:23 Enabled. ] [ Nov 8 10:30:23 Rereading configuration. ] [ Nov 8 10:30:24 Rereading configuration. ] [ Nov 8 10:30:28 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: f85cf0dc3631e2f0c13384787efb0efc1b642162\nCommit timestamp: 2026-10-10T00:38:51.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:28.146712545Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904} {"msg":"dataset: \"oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:28.146926467Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904} {"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:28.146945397Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:28.146955157Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:28.172214309Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:28.327319972Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.771029885Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:50864"} {"msg":"region 0cb19f7d-de94-45de-ab8d-2a727cfe8a0c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.801739823Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.808125298Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.812018729Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"uri":"/crucible/0/regions","method":"POST","req_id":"300563aa-1ba7-4f29-b08c-a70ed8dd2abb","remote_addr":"[fd00:1122:3344:101::b]:50864","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19053,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.814881178Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.814901088Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.872789547Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"region":"0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\" \"--data\" \"/data/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.872834508Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"region":"0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:30:59.999910502Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"region":"0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","component":"worker"} {"msg":"region 0cb19f7d-de94-45de-ab8d-2a727cfe8a0c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.001895329Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.002025268Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.00237376Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.008706293Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.01617914Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.017753036Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.018518114Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.018596635Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.018631155Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.018672726Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.021324523Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.024187613Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:31:00.115036472Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"uri":"/crucible/0/regions","method":"POST","req_id":"7299cf9a-fd35-40ff-b22e-0a5b6264c2f6","remote_addr":"[fd00:1122:3344:101::b]:50864","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":98,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.287912837Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:62220"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.288009298Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"uri":"/crucible/0/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","method":"GET","req_id":"69a39df2-9c28-4690-ba53-718bb91e4b04","remote_addr":"[fd00:1122:3344:101::b]:62220","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":43,"response_code":200} {"msg":"path is \"/data/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.295142032Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"dataset is oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.301300676Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"snapshot is oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c@ed7902d0-122b-4707-9c21-8a1b4bc73dd4","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.307306367Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\", \"ed7902d0-122b-4707-9c21-8a1b4bc73dd4\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.307340738Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.318126219Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"uri":"/crucible/0/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c/snapshots/ed7902d0-122b-4707-9c21-8a1b4bc73dd4","method":"GET","req_id":"837dd100-bbd0-436b-bd4e-12e05793302e","remote_addr":"[fd00:1122:3344:101::b]:62220","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26455,"response_code":200} {"msg":"path is \"/data/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.321467394Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"dataset is oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.326384085Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"snapshot is oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c@ed7902d0-122b-4707-9c21-8a1b4bc73dd4","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.332223135Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\", \"ed7902d0-122b-4707-9c21-8a1b4bc73dd4\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.332239135Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 0cb19f7d-de94-45de-ab8d-2a727cfe8a0c-ed7902d0-122b-4707-9c21-8a1b4bc73dd4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.33772138Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"applying SMF actions for region 0cb19f7d-de94-45de-ab8d-2a727cfe8a0c running snapshot ed7902d0-122b-4707-9c21-8a1b4bc73dd4 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.338704472Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.338718692Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"uri":"/crucible/0/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c/snapshots/ed7902d0-122b-4707-9c21-8a1b4bc73dd4/run","method":"POST","req_id":"92ab45ce-5b98-4be4-a3fc-155560274234","remote_addr":"[fd00:1122:3344:101::b]:62220","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":17278,"response_code":200} {"msg":"creating missing snapshot instance snapshot-0cb19f7d-de94-45de-ab8d-2a727cfe8a0c-ed7902d0-122b-4707-9c21-8a1b4bc73dd4","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.344723054Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-0cb19f7d-de94-45de-ab8d-2a727cfe8a0c-ed7902d0-122b-4707-9c21-8a1b4bc73dd4","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.349983039Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-0cb19f7d-de94-45de-ab8d-2a727cfe8a0c-ed7902d0-122b-4707-9c21-8a1b4bc73dd4","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.354779538Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c/.zfs/snapshot/ed7902d0-122b-4707-9c21-8a1b4bc73dd4","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.355051181Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.355085652Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.355132892Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.355170293Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.355213643Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.35784066Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.361038303Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"worker"} {"msg":"region 0cb19f7d-de94-45de-ab8d-2a727cfe8a0c running snapshot ed7902d0-122b-4707-9c21-8a1b4bc73dd4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.361054213Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"path is \"/data/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.562577477Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"dataset is oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.568061884Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"datafile"} {"msg":"snapshot is oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c@ed7902d0-122b-4707-9c21-8a1b4bc73dd4","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.572870074Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_2e2baada-8c2a-4ca2-bd05-2602fb9e9045/crucible/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c\", \"ed7902d0-122b-4707-9c21-8a1b4bc73dd4\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.572891934Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-11-08T10:42:04.578903096Z","hostname":"oxz_crucible_d5c2d595-547f-4f5b-b309-908ab0f35f84","pid":27904,"uri":"/crucible/0/regions/0cb19f7d-de94-45de-ab8d-2a727cfe8a0c/snapshots/ed7902d0-122b-4707-9c21-8a1b4bc73dd4/run","method":"POST","req_id":"fcba4f9c-90da-4a1e-93f8-9ac1affee5f5","remote_addr":"[fd00:1122:3344:101::b]:62220","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":16367,"response_code":200}