[ Aug  8 23:00:12 Enabled. ]
[ Aug  8 23:00:12 Rereading configuration. ]
[ Aug  8 23:00:13 Rereading configuration. ]
[ Aug  8 23:00:15 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7a4f86e0-be34-44ec-95ad-0b2dbb70721a/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7a4f86e0-be34-44ec-95ad-0b2dbb70721a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.374281141Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.375386976Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.375400406Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.396086745Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.55054279Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.462089325Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62380"}
{"msg":"region c32ac66e-b024-41c2-8fce-e980a217e4ea state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.490593233Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.499345445Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.503819357Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions","method":"POST","req_id":"fe9f4abc-35ba-41bf-b421-ae41aac75521","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22702,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_7a4f86e0-be34-44ec-95ad-0b2dbb70721a/crucible/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.51550827Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7a4f86e0-be34-44ec-95ad-0b2dbb70721a/crucible/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.515530461Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c32ac66e-b024-41c2-8fce-e980a217e4ea\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.568267606Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"region":"c32ac66e-b024-41c2-8fce-e980a217e4ea","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c32ac66e-b024-41c2-8fce-e980a217e4ea\" \"--data\" \"/data/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.570397476Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"region":"c32ac66e-b024-41c2-8fce-e980a217e4ea","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.767242373Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions","method":"POST","req_id":"db7225a3-3a45-4c36-8079-c44e1cdc1bf3","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.861720211Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"region":"c32ac66e-b024-41c2-8fce-e980a217e4ea","component":"worker"}
{"msg":"region c32ac66e-b024-41c2-8fce-e980a217e4ea state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.861763042Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.861870753Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c32ac66e-b024-41c2-8fce-e980a217e4ea","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.862278239Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c32ac66e-b024-41c2-8fce-e980a217e4ea","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.867132987Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.884247086Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c32ac66e-b024-41c2-8fce-e980a217e4ea","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.895391661Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.898456804Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.900867967Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.900916288Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.900953739Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.904457378Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.908457964Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:50.510542365Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions","method":"POST","req_id":"70015b55-1f1d-4c02-a541-634ffe8c1cd0","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":641,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.935629113Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","method":"GET","req_id":"cc135b1c-b35c-43f4-bd94-a36d387028aa","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.937986676Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"datafile"}
{"msg":"dataset is oxp_7a4f86e0-be34-44ec-95ad-0b2dbb70721a/crucible/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.947791333Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"datafile"}
{"msg":"region c32ac66e-b024-41c2-8fce-e980a217e4ea state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.956301541Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.956472484Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.956555685Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","method":"DELETE","req_id":"930c7271-4d37-4481-8045-79e39a82931e","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":18611,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-c32ac66e-b024-41c2-8fce-e980a217e4ea (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.957529418Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.958989119Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","method":"GET","req_id":"e34bc404-07d4-4dc4-a28a-f098c3c3ea48","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.9662614Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7a4f86e0-be34-44ec-95ad-0b2dbb70721a/crucible/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.975690142Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"region":"c32ac66e-b024-41c2-8fce-e980a217e4ea","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:20.014192709Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","method":"GET","req_id":"a66ca136-e7d0-422c-88a5-58fe2c66f429","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":615,"response_code":"200"}
{"msg":"region c32ac66e-b024-41c2-8fce-e980a217e4ea state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:20.033040852Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:20.17049632Z","hostname":"oxz_crucible_9349159c-07d1-4421-84f3-e5eb72dfa978","pid":24987,"uri":"/crucible/0/regions/c32ac66e-b024-41c2-8fce-e980a217e4ea","method":"GET","req_id":"ac304d3e-533e-47bb-8c77-0ed25c08a398","remote_addr":"[fd00:1122:3344:101::a]:62380","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27,"response_code":"200"}