[ Oct 30 17:58:56 Enabled. ]
[ Oct 30 17:58:56 Rereading configuration. ]
[ Oct 30 17:58:58 Rereading configuration. ]
[ Oct 30 17:59:01 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4732e2a7-63e2-49f2-b8c6-46c83e797180/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: f85cf0dc3631e2f0c13384787efb0efc1b642162\nCommit timestamp: 2026-10-10T00:38:51.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:01.661615292Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807}
{"msg":"dataset: \"oxp_4732e2a7-63e2-49f2-b8c6-46c83e797180/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:01.661814476Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:01.661825987Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:01.661831907Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:01.678472896Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:01.748593209Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.179907972Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:65223"}
{"msg":"region b68bb896-776d-445d-b9f0-3a052c6cb3bb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.207875319Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.214906242Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.219795242Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"uri":"/crucible/0/regions","method":"POST","req_id":"da8116a3-8ad7-4b6a-a512-9fa47c375d25","remote_addr":"[fd00:1122:3344:101::b]:65223","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":16779,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_4732e2a7-63e2-49f2-b8c6-46c83e797180/crucible/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.227465711Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_4732e2a7-63e2-49f2-b8c6-46c83e797180/crucible/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.227492061Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b68bb896-776d-445d-b9f0-3a052c6cb3bb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.31739253Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"region":"b68bb896-776d-445d-b9f0-3a052c6cb3bb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b68bb896-776d-445d-b9f0-3a052c6cb3bb\" \"--data\" \"/data/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.317504603Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"region":"b68bb896-776d-445d-b9f0-3a052c6cb3bb","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.387913403Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"uri":"/crucible/0/regions","method":"POST","req_id":"72f7b9f9-5a8a-4c4a-bb43-4f31cededb94","remote_addr":"[fd00:1122:3344:101::b]:65223","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":73,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.458964859Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"region":"b68bb896-776d-445d-b9f0-3a052c6cb3bb","component":"worker"}
{"msg":"region b68bb896-776d-445d-b9f0-3a052c6cb3bb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.460984449Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.461285326Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b68bb896-776d-445d-b9f0-3a052c6cb3bb","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.461607094Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b68bb896-776d-445d-b9f0-3a052c6cb3bb","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.469495628Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.483877901Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b68bb896-776d-445d-b9f0-3a052c6cb3bb","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.487284695Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.488010973Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.488071474Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.488113075Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.488147736Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.492434151Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.496672906Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:26.796777161Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"uri":"/crucible/0/regions","method":"POST","req_id":"b6e3fe37-e3cb-4060-929e-82230259c76b","remote_addr":"[fd00:1122:3344:101::b]:65223","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":82,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.312480075Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"uri":"/crucible/0/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","method":"GET","req_id":"c6398299-95b9-4c42-bd86-c2021edbd603","remote_addr":"[fd00:1122:3344:101::b]:65223","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":200}
{"msg":"path is \"/data/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.31637378Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"datafile"}
{"msg":"dataset is oxp_4732e2a7-63e2-49f2-b8c6-46c83e797180/crucible/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.323632899Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"datafile"}
{"msg":"region b68bb896-776d-445d-b9f0-3a052c6cb3bb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.330235971Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.330436636Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.330494207Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"uri":"/crucible/0/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","method":"DELETE","req_id":"20351ecf-565b-49b1-a165-02bb4ea8db7e","remote_addr":"[fd00:1122:3344:101::b]:65223","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14121,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-b68bb896-776d-445d-b9f0-3a052c6cb3bb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.331605675Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.333625804Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"uri":"/crucible/0/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","method":"GET","req_id":"8ee06625-f544-4afd-9566-f1f169e574a2","remote_addr":"[fd00:1122:3344:101::b]:65223","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":39,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.342937833Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_4732e2a7-63e2-49f2-b8c6-46c83e797180/crucible/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.351130305Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"region":"b68bb896-776d-445d-b9f0-3a052c6cb3bb","component":"worker"}
{"msg":"region b68bb896-776d-445d-b9f0-3a052c6cb3bb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.423045761Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-30T17:59:27.462089911Z","hostname":"oxz_crucible_993ad0b9-ed5f-44d4-92fd-55ca1182198d","pid":23807,"uri":"/crucible/0/regions/b68bb896-776d-445d-b9f0-3a052c6cb3bb","method":"GET","req_id":"dd7d1c21-dcf0-490b-8cb3-c9c10d97a917","remote_addr":"[fd00:1122:3344:101::b]:65223","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":51,"response_code":200}