[ Oct 14 01:29:12 Enabled. ]
[ Oct 14 01:29:12 Rereading configuration. ]
[ Oct 14 01:29:15 Rereading configuration. ]
[ Oct 14 01:29:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: f85cf0dc3631e2f0c13384787efb0efc1b642162\nCommit timestamp: 2026-10-10T00:38:51.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:21.479636521Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704}
{"msg":"dataset: \"oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:21.479826025Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:21.479837025Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:21.479842245Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:21.499057013Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:21.60685402Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.254590082Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:45936"}
{"msg":"region 36457f46-e771-4eac-9b2a-34e832e1c571 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.263453846Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.267806622Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.267863163Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions","method":"POST","req_id":"648faaa9-bf93-45d3-9007-798a1f3e4c73","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":7165,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible/regions/36457f46-e771-4eac-9b2a-34e832e1c571","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.273797909Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible/regions/36457f46-e771-4eac-9b2a-34e832e1c571","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.27381741Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"36457f46-e771-4eac-9b2a-34e832e1c571\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/36457f46-e771-4eac-9b2a-34e832e1c571\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.362941389Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"region":"36457f46-e771-4eac-9b2a-34e832e1c571","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"36457f46-e771-4eac-9b2a-34e832e1c571\" \"--data\" \"/data/regions/36457f46-e771-4eac-9b2a-34e832e1c571\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.363021281Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"region":"36457f46-e771-4eac-9b2a-34e832e1c571","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.508919136Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"region":"36457f46-e771-4eac-9b2a-34e832e1c571","component":"worker"}
{"msg":"region 36457f46-e771-4eac-9b2a-34e832e1c571 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.508990468Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.509217432Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-36457f46-e771-4eac-9b2a-34e832e1c571","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.509543858Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-36457f46-e771-4eac-9b2a-34e832e1c571","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.515903693Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.532566231Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-36457f46-e771-4eac-9b2a-34e832e1c571","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.539904285Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/36457f46-e771-4eac-9b2a-34e832e1c571","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.540651589Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.540712541Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.540750621Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.540788852Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.546230479Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.548701478Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions","method":"POST","req_id":"8656cf44-29ef-4b19-9744-f4946a8a37f3","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":88,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:44.553833388Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.480538148Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions/36457f46-e771-4eac-9b2a-34e832e1c571","method":"GET","req_id":"8f4bc97f-5ee9-430f-a1a2-b21efab284d0","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":74,"response_code":200}
{"msg":"path is \"/data/regions/36457f46-e771-4eac-9b2a-34e832e1c571\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.483071638Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"dataset is oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible/regions/36457f46-e771-4eac-9b2a-34e832e1c571","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.491203738Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"region 36457f46-e771-4eac-9b2a-34e832e1c571 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.499693135Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.499897789Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.499907049Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions/36457f46-e771-4eac-9b2a-34e832e1c571","method":"DELETE","req_id":"30bc444a-b0a1-48c4-8582-2c20ac570776","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":16877,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-36457f46-e771-4eac-9b2a-34e832e1c571 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.500922869Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.505479928Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions/36457f46-e771-4eac-9b2a-34e832e1c571","method":"GET","req_id":"59ecc83d-9690-4fb1-a4e7-cd5214a3363e","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.505873816Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible/regions/36457f46-e771-4eac-9b2a-34e832e1c571\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.514857422Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"region":"36457f46-e771-4eac-9b2a-34e832e1c571","component":"worker"}
{"msg":"region 36457f46-e771-4eac-9b2a-34e832e1c571 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.601971204Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:45.626698985Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions/36457f46-e771-4eac-9b2a-34e832e1c571","method":"GET","req_id":"28d1771a-5d79-4ff1-a5f4-512834d2eefa","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"region 6e366316-ed98-4653-9a74-ab9be612c9a6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.655026784Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.65536236Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions","method":"POST","req_id":"18ee2596-3c5f-4c9a-a766-0c2b56962761","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":370,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.655442652Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible/regions/6e366316-ed98-4653-9a74-ab9be612c9a6","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.661045702Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_092244ff-277f-4ddd-a58f-23dfc474a683/crucible/regions/6e366316-ed98-4653-9a74-ab9be612c9a6","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.661059502Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6e366316-ed98-4653-9a74-ab9be612c9a6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6e366316-ed98-4653-9a74-ab9be612c9a6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.732562436Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"region":"6e366316-ed98-4653-9a74-ab9be612c9a6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6e366316-ed98-4653-9a74-ab9be612c9a6\" \"--data\" \"/data/regions/6e366316-ed98-4653-9a74-ab9be612c9a6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.732636508Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"region":"6e366316-ed98-4653-9a74-ab9be612c9a6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.806326985Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"region":"6e366316-ed98-4653-9a74-ab9be612c9a6","component":"worker"}
{"msg":"region 6e366316-ed98-4653-9a74-ab9be612c9a6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.806392857Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.806605141Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6e366316-ed98-4653-9a74-ab9be612c9a6","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.807833525Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6e366316-ed98-4653-9a74-ab9be612c9a6","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.810856044Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.819044785Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6e366316-ed98-4653-9a74-ab9be612c9a6","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.822624155Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6e366316-ed98-4653-9a74-ab9be612c9a6","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.822909411Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.822959502Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.823003473Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.823053834Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.828345177Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.833980038Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-14T01:29:48.984417803Z","hostname":"oxz_crucible_9091635f-135d-4564-a2f7-f2fd6fc025ec","pid":25704,"uri":"/crucible/0/regions","method":"POST","req_id":"6a038d8c-c64a-43ed-8b43-4a7885f3b033","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200}