[ Jun 27 21:49:54 Enabled. ]
[ Jun 27 21:49:54 Rereading configuration. ]
[ Jun 27 21:49:57 Rereading configuration. ]
[ Jun 27 21:50:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0c8a1506-92f8-47f9-bf19-87d5fddc97aa/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0c8a1506-92f8-47f9-bf19-87d5fddc97aa/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:04.034933979Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:04.0371357Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:04.03714652Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:04.047558107Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:04.200953913Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.351845684Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46014"}
{"msg":"region 481ab2d7-9b63-489e-90fa-1767823bc469 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.379082213Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.390421921Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"uri":"/crucible/0/regions","method":"POST","req_id":"6c8617ea-86d2-47c2-b5ba-b6234e00bda5","remote_addr":"[fd00:1122:3344:101::a]:46014","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24141,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.390451552Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_0c8a1506-92f8-47f9-bf19-87d5fddc97aa/crucible/regions/481ab2d7-9b63-489e-90fa-1767823bc469","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.406273502Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0c8a1506-92f8-47f9-bf19-87d5fddc97aa/crucible/regions/481ab2d7-9b63-489e-90fa-1767823bc469","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.406328323Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"481ab2d7-9b63-489e-90fa-1767823bc469\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/481ab2d7-9b63-489e-90fa-1767823bc469\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.451502529Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"region":"481ab2d7-9b63-489e-90fa-1767823bc469","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"481ab2d7-9b63-489e-90fa-1767823bc469\" \"--data\" \"/data/regions/481ab2d7-9b63-489e-90fa-1767823bc469\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.453792061Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"region":"481ab2d7-9b63-489e-90fa-1767823bc469","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.686252031Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"uri":"/crucible/0/regions","method":"POST","req_id":"b2f810d9-aa9e-45b0-ab21-5631b215ae1d","remote_addr":"[fd00:1122:3344:101::a]:46014","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.724599622Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"region":"481ab2d7-9b63-489e-90fa-1767823bc469","component":"worker"}
{"msg":"region 481ab2d7-9b63-489e-90fa-1767823bc469 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.724625192Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.724754866Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-481ab2d7-9b63-489e-90fa-1767823bc469","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.725168497Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-481ab2d7-9b63-489e-90fa-1767823bc469","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.732762213Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.749766785Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-481ab2d7-9b63-489e-90fa-1767823bc469","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.753178187Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/481ab2d7-9b63-489e-90fa-1767823bc469","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.753932558Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.756162579Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.75622276Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.756262591Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.766131609Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:41.77315283Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:42.363565305Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"uri":"/crucible/0/regions","method":"POST","req_id":"e35414ae-e7ae-4902-8b89-5eade91fd9e6","remote_addr":"[fd00:1122:3344:101::a]:46014","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.462392726Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"uri":"/crucible/0/regions/481ab2d7-9b63-489e-90fa-1767823bc469","method":"GET","req_id":"933e209c-a716-470c-ad2f-e04086001ebf","remote_addr":"[fd00:1122:3344:101::a]:46014","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"path is \"/data/regions/481ab2d7-9b63-489e-90fa-1767823bc469\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.467019602Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"datafile"}
{"msg":"dataset is oxp_0c8a1506-92f8-47f9-bf19-87d5fddc97aa/crucible/regions/481ab2d7-9b63-489e-90fa-1767823bc469","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.477687991Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"datafile"}
{"msg":"region 481ab2d7-9b63-489e-90fa-1767823bc469 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.485911034Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.486141341Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.486233953Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"uri":"/crucible/0/regions/481ab2d7-9b63-489e-90fa-1767823bc469","method":"DELETE","req_id":"a81fd9a2-bc23-47c0-81b2-038fc29073be","remote_addr":"[fd00:1122:3344:101::a]:46014","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":21125,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-481ab2d7-9b63-489e-90fa-1767823bc469 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.48760173Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.48942573Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"uri":"/crucible/0/regions/481ab2d7-9b63-489e-90fa-1767823bc469","method":"GET","req_id":"d6f4783f-e404-43ca-8a51-cb6b7ccf93ae","remote_addr":"[fd00:1122:3344:101::a]:46014","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":482,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.491483156Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0c8a1506-92f8-47f9-bf19-87d5fddc97aa/crucible/regions/481ab2d7-9b63-489e-90fa-1767823bc469\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.50122871Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"region":"481ab2d7-9b63-489e-90fa-1767823bc469","component":"worker"}
{"msg":"region 481ab2d7-9b63-489e-90fa-1767823bc469 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.545511762Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:57:05.64012786Z","hostname":"oxz_crucible_609b023a-f0db-4bc5-b3e3-395c7ebd6de9","pid":21125,"uri":"/crucible/0/regions/481ab2d7-9b63-489e-90fa-1767823bc469","method":"GET","req_id":"59540cd3-fba1-4ee1-ab83-cb0943de4aee","remote_addr":"[fd00:1122:3344:101::a]:46014","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":53,"response_code":"200"}