[ Jul 11 17:58:18 Enabled. ]
[ Jul 11 17:58:18 Rereading configuration. ]
[ Jul 11 17:58:20 Rereading configuration. ]
[ Jul 11 17:58:25 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fad2fb3d-bd64-4e85-9580-dff5316a3f62/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 2bfe090eb5318ec8c467157018db2429d4df535b\nCommit timestamp: 2026-06-25T19:33:01.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:58:25.126243124Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890}
{"msg":"dataset: \"oxp_fad2fb3d-bd64-4e85-9580-dff5316a3f62/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:58:25.126406441Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:58:25.126449012Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:58:25.126455726Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:58:25.141864933Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:58:25.299541578Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.421375181Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36725"}
{"msg":"region 331fc3d9-3867-40ee-8933-dd0f265c2b54 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.449891976Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.454608486Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.456825929Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions","method":"POST","req_id":"07109dd4-3b43-4a2d-afee-1f6f9f5989ec","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":15488,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_fad2fb3d-bd64-4e85-9580-dff5316a3f62/crucible/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.463081448Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fad2fb3d-bd64-4e85-9580-dff5316a3f62/crucible/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.463110619Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"331fc3d9-3867-40ee-8933-dd0f265c2b54\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.516779656Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"region":"331fc3d9-3867-40ee-8933-dd0f265c2b54","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"331fc3d9-3867-40ee-8933-dd0f265c2b54\" \"--data\" \"/data/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.518752313Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"region":"331fc3d9-3867-40ee-8933-dd0f265c2b54","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.624381104Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions","method":"POST","req_id":"1e899e7b-6ced-4f49-9553-3650afab2276","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.657598869Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"region":"331fc3d9-3867-40ee-8933-dd0f265c2b54","component":"worker"}
{"msg":"region 331fc3d9-3867-40ee-8933-dd0f265c2b54 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.661433722Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.661584205Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-331fc3d9-3867-40ee-8933-dd0f265c2b54","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.663871779Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-331fc3d9-3867-40ee-8933-dd0f265c2b54","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.671346302Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.686203146Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-331fc3d9-3867-40ee-8933-dd0f265c2b54","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.691060049Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.691792123Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.6937276Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.693757031Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.693800252Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.7047031Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:15.714467937Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.368262251Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions","method":"POST","req_id":"0574beb7-5614-4f56-83e5-2ad025d3246a","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":75,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.857623031Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","method":"GET","req_id":"db423513-cee9-4350-bb00-1cd8648b603d","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"path is \"/data/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.861947834Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"datafile"}
{"msg":"dataset is oxp_fad2fb3d-bd64-4e85-9580-dff5316a3f62/crucible/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.869718422Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"datafile"}
{"msg":"region 331fc3d9-3867-40ee-8933-dd0f265c2b54 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.876718636Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.876836998Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.876872519Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","method":"DELETE","req_id":"461b5f55-40f5-4021-ad6f-5f6ceee9d8bc","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14941,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-331fc3d9-3867-40ee-8933-dd0f265c2b54 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.877861288Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.881705521Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","method":"GET","req_id":"d41f2a8e-d1ae-4bdf-a07d-11fafa378e68","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.883001006Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fad2fb3d-bd64-4e85-9580-dff5316a3f62/crucible/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.891727393Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"region":"331fc3d9-3867-40ee-8933-dd0f265c2b54","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.947982189Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","method":"GET","req_id":"aff63089-c239-4d4c-8aa9-a36a00805c84","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"region 331fc3d9-3867-40ee-8933-dd0f265c2b54 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:16.954441803Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-11T17:59:17.049014171Z","hostname":"oxz_crucible_4fa85367-3a9a-4666-8e54-5c68d6c70785","pid":23890,"uri":"/crucible/0/regions/331fc3d9-3867-40ee-8933-dd0f265c2b54","method":"GET","req_id":"a968fd57-ccea-491d-a51d-9877504833ff","remote_addr":"[fd00:1122:3344:101::a]:36725","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":52,"response_code":200}