[ Nov 19 19:40:34 Enabled. ]
[ Nov 19 19:40:34 Rereading configuration. ]
[ Nov 19 19:40:35 Rereading configuration. ]
[ Nov 19 19:40:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_43c663f2-b051-403e-bcad-e419348a9017/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_43c663f2-b051-403e-bcad-e419348a9017/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:40:38.868046024Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:40:38.868336032Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:40:38.868345892Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:40:38.887139681Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:40:38.998020475Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:05.893528534Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:45374"}
{"msg":"region 975b82b9-496d-4127-942d-bd6be09a4f4d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:05.933848146Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:05.940970769Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"uri":"/crucible/0/regions","method":"POST","req_id":"0d11a212-4dc0-4624-b829-57b34e41f977","remote_addr":"[fd00:1122:3344:101::b]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21108,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:05.941707758Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_43c663f2-b051-403e-bcad-e419348a9017/crucible/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:05.951990407Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_43c663f2-b051-403e-bcad-e419348a9017/crucible/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:05.952052979Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"975b82b9-496d-4127-942d-bd6be09a4f4d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/975b82b9-496d-4127-942d-bd6be09a4f4d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.074937307Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"region":"975b82b9-496d-4127-942d-bd6be09a4f4d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"975b82b9-496d-4127-942d-bd6be09a4f4d\" \"--data\" \"/data/regions/975b82b9-496d-4127-942d-bd6be09a4f4d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.078969116Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"region":"975b82b9-496d-4127-942d-bd6be09a4f4d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.262706152Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"region":"975b82b9-496d-4127-942d-bd6be09a4f4d","component":"worker"}
{"msg":"region 975b82b9-496d-4127-942d-bd6be09a4f4d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.262793845Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.262969449Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-975b82b9-496d-4127-942d-bd6be09a4f4d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.263444732Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-975b82b9-496d-4127-942d-bd6be09a4f4d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.27851594Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.308723509Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.315547753Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"uri":"/crucible/0/regions","method":"POST","req_id":"4fda66d8-1cb8-4c42-80cc-844534676b88","remote_addr":"[fd00:1122:3344:101::b]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-975b82b9-496d-4127-942d-bd6be09a4f4d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.327166618Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.327910158Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.334160987Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.334219229Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.33426791Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.365958199Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:06.387243105Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.478866599Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"uri":"/crucible/0/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","method":"GET","req_id":"47d28e93-2388-4eba-a4d0-1ab533388db6","remote_addr":"[fd00:1122:3344:101::b]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/975b82b9-496d-4127-942d-bd6be09a4f4d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.482293142Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"datafile"}
{"msg":"dataset is oxp_43c663f2-b051-403e-bcad-e419348a9017/crucible/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.493892086Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"datafile"}
{"msg":"region 975b82b9-496d-4127-942d-bd6be09a4f4d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.506027975Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.506260261Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-975b82b9-496d-4127-942d-bd6be09a4f4d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.507348671Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.510191848Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"uri":"/crucible/0/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","method":"DELETE","req_id":"702da73c-6467-4827-943b-599e10bde67e","remote_addr":"[fd00:1122:3344:101::b]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27925,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.514200706Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"uri":"/crucible/0/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","method":"GET","req_id":"e24c4af1-ac63-4418-953b-19df03944086","remote_addr":"[fd00:1122:3344:101::b]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":42,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.523869518Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_43c663f2-b051-403e-bcad-e419348a9017/crucible/regions/975b82b9-496d-4127-942d-bd6be09a4f4d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.537928989Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"region":"975b82b9-496d-4127-942d-bd6be09a4f4d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.580125742Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"uri":"/crucible/0/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","method":"GET","req_id":"d1150d52-4664-41e6-aa86-bdee5b5ad890","remote_addr":"[fd00:1122:3344:101::b]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"region 975b82b9-496d-4127-942d-bd6be09a4f4d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.683796689Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:41:07.701950531Z","hostname":"oxz_crucible_32b0278d-b5e0-48f7-97b1-7e59a0a057ad","pid":22447,"uri":"/crucible/0/regions/975b82b9-496d-4127-942d-bd6be09a4f4d","method":"GET","req_id":"644d82c2-48da-4d29-9ba9-ad8c3efd5623","remote_addr":"[fd00:1122:3344:101::b]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":45,"response_code":"200"}