[ Oct 16 19:55:23 Enabled. ]
[ Oct 16 19:55:23 Rereading configuration. ]
[ Oct 16 19:55:24 Rereading configuration. ]
[ Oct 16 19:55:26 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5d388581-9a58-40b6-9601-b0ab9895a4b3/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_5d388581-9a58-40b6-9601-b0ab9895a4b3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:55:26.567032505Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:55:26.568919648Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:55:26.568931598Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:55:26.586131603Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:55:26.723706108Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.041235557Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64144"}
{"msg":"region 8b06c86a-3035-4e5e-935b-a79bfd674671 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.068870961Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.071342188Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"uri":"/crucible/0/regions","method":"POST","req_id":"9e526a27-02e1-4412-84de-bd2aa7fcecbd","remote_addr":"[fd00:1122:3344:101::b]:64144","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":10658,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.072585006Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_5d388581-9a58-40b6-9601-b0ab9895a4b3/crucible/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.081737126Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_5d388581-9a58-40b6-9601-b0ab9895a4b3/crucible/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.081794047Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8b06c86a-3035-4e5e-935b-a79bfd674671\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8b06c86a-3035-4e5e-935b-a79bfd674671\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.162656442Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"region":"8b06c86a-3035-4e5e-935b-a79bfd674671","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8b06c86a-3035-4e5e-935b-a79bfd674671\" \"--data\" \"/data/regions/8b06c86a-3035-4e5e-935b-a79bfd674671\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.166700535Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"region":"8b06c86a-3035-4e5e-935b-a79bfd674671","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.285704465Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"region":"8b06c86a-3035-4e5e-935b-a79bfd674671","component":"worker"}
{"msg":"region 8b06c86a-3035-4e5e-935b-a79bfd674671 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.285797187Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.286013061Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8b06c86a-3035-4e5e-935b-a79bfd674671","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.288366946Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8b06c86a-3035-4e5e-935b-a79bfd674671","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.292638833Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.309556321Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8b06c86a-3035-4e5e-935b-a79bfd674671","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.313600044Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.319686304Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.319754315Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.319798656Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.319851958Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.324914774Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.329603681Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:19.356791235Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"uri":"/crucible/0/regions","method":"POST","req_id":"e32be7a9-f896-4689-aa4e-b855cc4c26e4","remote_addr":"[fd00:1122:3344:101::b]:64144","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.323716103Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"uri":"/crucible/0/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","method":"GET","req_id":"92f6eb65-bd45-4b5a-986b-e703f59f5933","remote_addr":"[fd00:1122:3344:101::b]:64144","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/8b06c86a-3035-4e5e-935b-a79bfd674671\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.326770953Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"datafile"}
{"msg":"dataset is oxp_5d388581-9a58-40b6-9601-b0ab9895a4b3/crucible/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.337646053Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"datafile"}
{"msg":"region 8b06c86a-3035-4e5e-935b-a79bfd674671 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.347291284Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.347501749Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.347573331Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"uri":"/crucible/0/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","method":"DELETE","req_id":"14bb63c5-21fe-4ee3-af81-df5a4fd7f644","remote_addr":"[fd00:1122:3344:101::b]:64144","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20827,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-8b06c86a-3035-4e5e-935b-a79bfd674671 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.348581404Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.350737643Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"uri":"/crucible/0/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","method":"GET","req_id":"fdc38899-78ae-4e50-a44e-14013161db0e","remote_addr":"[fd00:1122:3344:101::b]:64144","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":37,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.354640123Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5d388581-9a58-40b6-9601-b0ab9895a4b3/crucible/regions/8b06c86a-3035-4e5e-935b-a79bfd674671\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.367939498Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"region":"8b06c86a-3035-4e5e-935b-a79bfd674671","component":"worker"}
{"msg":"region 8b06c86a-3035-4e5e-935b-a79bfd674671 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.430738118Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T19:56:20.485070724Z","hostname":"oxz_crucible_8a54e004-7b79-41d6-80d3-663fbddc9249","pid":20751,"uri":"/crucible/0/regions/8b06c86a-3035-4e5e-935b-a79bfd674671","method":"GET","req_id":"effbdfe3-f43b-440b-96d9-3eb1f36c4705","remote_addr":"[fd00:1122:3344:101::b]:64144","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":"200"}