[ Jan 17 07:28:50 Enabled. ]
[ Jan 17 07:28:50 Rereading configuration. ]
[ Jan 17 07:28:53 Rereading configuration. ]
[ Jan 17 07:28:59 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ade9310a-246d-42ae-ac99-e5fa7edb32cd/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ade9310a-246d-42ae-ac99-e5fa7edb32cd/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:28:59.958454816Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:28:59.959312523Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:28:59.959320453Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:28:59.977565777Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:00.270383084Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.372538872Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:37649"}
{"msg":"region 46ff8ead-fe0b-41cd-8ac4-29bc7d5347be state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.396225042Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.398353284Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions","method":"POST","req_id":"80c4acd1-461b-4a2f-9a6c-510baa17dafa","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":8603,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.399037337Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_ade9310a-246d-42ae-ac99-e5fa7edb32cd/crucible/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.410708353Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ade9310a-246d-42ae-ac99-e5fa7edb32cd/crucible/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.410745654Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"46ff8ead-fe0b-41cd-8ac4-29bc7d5347be\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.484082198Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"region":"46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"46ff8ead-fe0b-41cd-8ac4-29bc7d5347be\" \"--data\" \"/data/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.486046977Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"region":"46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.588072818Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions","method":"POST","req_id":"1f8820fa-44bf-42e5-a9ad-01a02df3c04e","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.610724798Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"region":"46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","component":"worker"}
{"msg":"region 46ff8ead-fe0b-41cd-8ac4-29bc7d5347be state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.610762459Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.610888412Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.611439362Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.61699276Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.630520303Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.639817794Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.640318463Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.640452476Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.640495936Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.640535607Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.644035745Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:55.648275388Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.200031094Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions","method":"POST","req_id":"6cd97bb7-3213-4d8e-9fa2-b3697513da4e","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.525096768Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","method":"GET","req_id":"3a48c1d2-f945-453c-9558-045b9ff2024c","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"path is \"/data/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.527724869Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"datafile"}
{"msg":"dataset is oxp_ade9310a-246d-42ae-ac99-e5fa7edb32cd/crucible/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.539392515Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"datafile"}
{"msg":"region 46ff8ead-fe0b-41cd-8ac4-29bc7d5347be state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.549625894Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.549781517Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-46ff8ead-fe0b-41cd-8ac4-29bc7d5347be (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.550824517Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.551869708Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","method":"DELETE","req_id":"91d8f822-f5d5-45f5-92e3-627c7ea09731","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24191,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.554809045Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","method":"GET","req_id":"d6e38fbb-8541-4f44-8c9c-c1e855e2735e","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.555355596Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ade9310a-246d-42ae-ac99-e5fa7edb32cd/crucible/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.56692904Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"region":"46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.627618569Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","method":"GET","req_id":"9c07c520-89f5-4263-81ef-6f1a5387e9b7","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region 46ff8ead-fe0b-41cd-8ac4-29bc7d5347be state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.638027681Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T07:29:56.735029715Z","hostname":"oxz_crucible_d5a24395-204f-48e5-af46-fb7ee00e3945","pid":22630,"uri":"/crucible/0/regions/46ff8ead-fe0b-41cd-8ac4-29bc7d5347be","method":"GET","req_id":"8571094b-5e1a-48ca-a06f-3609df6d1894","remote_addr":"[fd00:1122:3344:101::b]:37649","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31,"response_code":"200"}