[ Nov  4 23:54:18 Enabled. ]
[ Nov  4 23:54:18 Rereading configuration. ]
[ Nov  4 23:54:19 Rereading configuration. ]
[ Nov  4 23:54:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7449cf8d-6d8a-4759-b257-0067c28bc087/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7449cf8d-6d8a-4759-b257-0067c28bc087/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:21.912609937Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:21.915532127Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:21.915550558Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:21.933662129Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:22.337322735Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.822290975Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49685"}
{"msg":"region e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.836983327Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.841880977Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions","method":"POST","req_id":"b0d5698a-4deb-4fdc-88b2-1b75ddbd1304","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":7632,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.841914058Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_7449cf8d-6d8a-4759-b257-0067c28bc087/crucible/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.852106017Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7449cf8d-6d8a-4759-b257-0067c28bc087/crucible/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.852182399Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.923427251Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"region":"e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2\" \"--data\" \"/data/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.925718408Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"region":"e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:48.999306709Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions","method":"POST","req_id":"05226d3e-ff47-4881-9c73-90da8f7b84f1","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.224111514Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"region":"e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","component":"worker"}
{"msg":"region e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.224189495Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.22439755Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.224812238Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.230540976Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.251560507Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.256620531Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.257432698Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.259804516Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.259878158Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.259918839Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.272496667Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.279353288Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:49.315594422Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions","method":"POST","req_id":"a47a85f7-6c40-4aad-afab-ab71bea2210b","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.04621883Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","method":"GET","req_id":"04c86abd-340e-467e-8d39-716039c86c90","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"path is \"/data/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.049074969Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"datafile"}
{"msg":"dataset is oxp_7449cf8d-6d8a-4759-b257-0067c28bc087/crucible/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.060565625Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"datafile"}
{"msg":"region e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.071154612Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.071368356Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.072439038Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.073927389Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","method":"DELETE","req_id":"741db7e7-9130-4d75-936a-ae7d12ebea13","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":24881,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.078555644Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","method":"GET","req_id":"21585ee4-e31b-49f0-abce-8787c880e511","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.080276249Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7449cf8d-6d8a-4759-b257-0067c28bc087/crucible/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.09118496Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"region":"e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.167151793Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","method":"GET","req_id":"84d0bb5a-657a-41ff-9868-5267e1b8853b","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":112,"response_code":"200"}
{"msg":"region e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.176372452Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-04T23:54:50.28687263Z","hostname":"oxz_crucible_3063cc09-aec0-4a7e-9f10-942e2ac49629","pid":22590,"uri":"/crucible/0/regions/e9fd5b94-923c-4ea1-9c49-f8ae8d197eb2","method":"GET","req_id":"97a3ae2e-2fad-4522-8c39-dd93b6770767","remote_addr":"[fd00:1122:3344:101::a]:49685","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":49,"response_code":"200"}