[ Sep  4 16:52:05 Enabled. ]
[ Sep  4 16:52:05 Rereading configuration. ]
[ Sep  4 16:52:06 Rereading configuration. ]
[ Sep  4 16:52:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:52:09.74790845Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:52:09.748865533Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:52:09.748876223Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:52:09.762906077Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:52:09.910015812Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.590378383Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:65350"}
{"msg":"region dd029375-3cae-4c2b-b7d7-c3b25507179d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.617707654Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.627114819Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.631238217Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"uri":"/crucible/0/regions","method":"POST","req_id":"34e30a57-c51c-454d-8725-836523b8f1ad","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":21777,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad/crucible/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.638320815Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad/crucible/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.638351796Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dd029375-3cae-4c2b-b7d7-c3b25507179d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.69013933Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"region":"dd029375-3cae-4c2b-b7d7-c3b25507179d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dd029375-3cae-4c2b-b7d7-c3b25507179d\" \"--data\" \"/data/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.692559178Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"region":"dd029375-3cae-4c2b-b7d7-c3b25507179d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.90629934Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"uri":"/crucible/0/regions","method":"POST","req_id":"9e7e132c-613a-495f-89c3-db2f7882acae","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.975719864Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"region":"dd029375-3cae-4c2b-b7d7-c3b25507179d","component":"worker"}
{"msg":"region dd029375-3cae-4c2b-b7d7-c3b25507179d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.975770746Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.975869218Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dd029375-3cae-4c2b-b7d7-c3b25507179d","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.976274407Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dd029375-3cae-4c2b-b7d7-c3b25507179d","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.981513122Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:16.999906621Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dd029375-3cae-4c2b-b7d7-c3b25507179d","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.009910219Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.01289495Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.015481732Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.015532523Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.015574264Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.019349504Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.02339712Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:17.345808272Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"uri":"/crucible/0/regions","method":"POST","req_id":"a0ef5057-acf0-4e81-8e6b-3094100d88cd","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":90,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.226057166Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"uri":"/crucible/0/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","method":"GET","req_id":"d6c21b18-853b-473f-a47e-57684134d358","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"path is \"/data/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.230134713Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"datafile"}
{"msg":"dataset is oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad/crucible/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.240709784Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"datafile"}
{"msg":"region dd029375-3cae-4c2b-b7d7-c3b25507179d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.249403422Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.249613347Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.249694779Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"uri":"/crucible/0/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","method":"DELETE","req_id":"b45617e0-d0b0-4337-b6f4-5490b1ec968d","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19599,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-dd029375-3cae-4c2b-b7d7-c3b25507179d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.250795585Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.252549647Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"uri":"/crucible/0/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","method":"GET","req_id":"73304db4-97c2-4eef-b00b-553b626b8742","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.25477773Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad/crucible/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.264524702Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"region":"dd029375-3cae-4c2b-b7d7-c3b25507179d","component":"worker"}
{"msg":"region dd029375-3cae-4c2b-b7d7-c3b25507179d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.327020381Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-04T16:58:45.335027772Z","hostname":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","pid":23790,"uri":"/crucible/0/regions/dd029375-3cae-4c2b-b7d7-c3b25507179d","method":"GET","req_id":"1d18dc15-07a1-48d1-85ba-e87d41467ec3","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":73,"response_code":"200"}