[ Mar 17 16:40:54 Enabled. ]
[ Mar 17 16:40:54 Rereading configuration. ]
[ Mar 17 16:40:57 Rereading configuration. ]
[ Mar 17 16:41:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_577d9173-6f77-4ceb-8241-9f67ddf95ddd/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_577d9173-6f77-4ceb-8241-9f67ddf95ddd/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.899052073Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.90217994Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.90218946Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.917950538Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.982417899Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.023965146Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60893"}
{"msg":"region de21d0a1-19aa-413f-acda-6ebba272e51a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.079361322Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.089208663Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"uri":"/crucible/0/regions","method":"POST","req_id":"3ccd51af-34d3-4683-85b7-f02b2d7b0af2","remote_addr":"[fd00:1122:3344:101::c]:60893","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29113,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.089231383Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_577d9173-6f77-4ceb-8241-9f67ddf95ddd/crucible/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.099387931Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_577d9173-6f77-4ceb-8241-9f67ddf95ddd/crucible/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.099436482Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"de21d0a1-19aa-413f-acda-6ebba272e51a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/de21d0a1-19aa-413f-acda-6ebba272e51a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.14558751Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"region":"de21d0a1-19aa-413f-acda-6ebba272e51a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"de21d0a1-19aa-413f-acda-6ebba272e51a\" \"--data\" \"/data/regions/de21d0a1-19aa-413f-acda-6ebba272e51a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.148005412Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"region":"de21d0a1-19aa-413f-acda-6ebba272e51a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.395134045Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"uri":"/crucible/0/regions","method":"POST","req_id":"51a0f88f-fadf-444d-9088-8879e062bff6","remote_addr":"[fd00:1122:3344:101::c]:60893","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":96,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.451624734Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"region":"de21d0a1-19aa-413f-acda-6ebba272e51a","component":"worker"}
{"msg":"region de21d0a1-19aa-413f-acda-6ebba272e51a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.451689405Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.451780687Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-de21d0a1-19aa-413f-acda-6ebba272e51a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.452267988Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-de21d0a1-19aa-413f-acda-6ebba272e51a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.457259675Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.469535848Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-de21d0a1-19aa-413f-acda-6ebba272e51a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.473122715Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.473700267Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.473760858Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.473802839Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.47384153Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.478013869Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.489582627Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.782310906Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"uri":"/crucible/0/regions","method":"POST","req_id":"77df104e-fe19-4923-a4c8-8f9751e1d6c3","remote_addr":"[fd00:1122:3344:101::c]:60893","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.189677481Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"uri":"/crucible/0/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","method":"GET","req_id":"aa5c3dbe-10d5-44e7-babd-e3bdac83e25a","remote_addr":"[fd00:1122:3344:101::c]:60893","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2339,"response_code":"200"}
{"msg":"path is \"/data/regions/de21d0a1-19aa-413f-acda-6ebba272e51a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.19288058Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"datafile"}
{"msg":"dataset is oxp_577d9173-6f77-4ceb-8241-9f67ddf95ddd/crucible/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.205214674Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"datafile"}
{"msg":"region de21d0a1-19aa-413f-acda-6ebba272e51a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.217420065Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.217560798Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.217612289Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"uri":"/crucible/0/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","method":"DELETE","req_id":"ac543b58-a952-432f-9c12-5a6dbd653d9d","remote_addr":"[fd00:1122:3344:101::c]:60893","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24779,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-de21d0a1-19aa-413f-acda-6ebba272e51a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.218620561Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.220828468Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"uri":"/crucible/0/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","method":"GET","req_id":"c489f4ba-82ec-4d2b-b108-2c4b7777da7e","remote_addr":"[fd00:1122:3344:101::c]:60893","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.22279496Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_577d9173-6f77-4ceb-8241-9f67ddf95ddd/crucible/regions/de21d0a1-19aa-413f-acda-6ebba272e51a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.234847729Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"region":"de21d0a1-19aa-413f-acda-6ebba272e51a","component":"worker"}
{"msg":"region de21d0a1-19aa-413f-acda-6ebba272e51a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.297600393Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.3619231Z","hostname":"oxz_crucible_340c7790-5b6a-4bb7-a53d-2a6a44ece820","pid":22434,"uri":"/crucible/0/regions/de21d0a1-19aa-413f-acda-6ebba272e51a","method":"GET","req_id":"07c331e5-9753-4137-b7e5-dcd023e37cf0","remote_addr":"[fd00:1122:3344:101::c]:60893","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}