[ Jan 29 07:09:07 Enabled. ]
[ Jan 29 07:09:07 Rereading configuration. ]
[ Jan 29 07:09:12 Rereading configuration. ]
[ Jan 29 07:09:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_db2b1192-66c6-4615-a94b-80071566494f/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_db2b1192-66c6-4615-a94b-80071566494f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:20.553791734Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:20.555961763Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:20.555975433Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:20.566183087Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:20.656659406Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.037620413Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52257"}
{"msg":"region 911c89d5-f1a1-4a75-b925-3a68a375b2e6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.077215656Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.079815285Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"uri":"/crucible/0/regions","method":"POST","req_id":"2358d38e-e574-4588-8250-3e6a77101ddc","remote_addr":"[fd00:1122:3344:101::a]:52257","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":15849,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.084202433Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_db2b1192-66c6-4615-a94b-80071566494f/crucible/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.093730389Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_db2b1192-66c6-4615-a94b-80071566494f/crucible/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.09377415Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"911c89d5-f1a1-4a75-b925-3a68a375b2e6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.150519664Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"region":"911c89d5-f1a1-4a75-b925-3a68a375b2e6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"911c89d5-f1a1-4a75-b925-3a68a375b2e6\" \"--data\" \"/data/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.152489966Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"region":"911c89d5-f1a1-4a75-b925-3a68a375b2e6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.32471953Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"uri":"/crucible/0/regions","method":"POST","req_id":"875dc397-b13c-4098-8b13-45e21ed3c87f","remote_addr":"[fd00:1122:3344:101::a]:52257","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.393139526Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"region":"911c89d5-f1a1-4a75-b925-3a68a375b2e6","component":"worker"}
{"msg":"region 911c89d5-f1a1-4a75-b925-3a68a375b2e6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.393176887Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.393307211Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-911c89d5-f1a1-4a75-b925-3a68a375b2e6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.393752013Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-911c89d5-f1a1-4a75-b925-3a68a375b2e6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.400526775Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.415486766Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-911c89d5-f1a1-4a75-b925-3a68a375b2e6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.429710218Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.430250103Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.432606616Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.432663978Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.432704079Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.437195239Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.442318447Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:09:59.774766522Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"uri":"/crucible/0/regions","method":"POST","req_id":"303510b4-7760-4757-ae5b-6a9d0a45b228","remote_addr":"[fd00:1122:3344:101::a]:52257","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.056878486Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"uri":"/crucible/0/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","method":"GET","req_id":"d85ac8b5-ca5f-47e6-80f3-0925979ff989","remote_addr":"[fd00:1122:3344:101::a]:52257","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"path is \"/data/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.060094082Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"datafile"}
{"msg":"dataset is oxp_db2b1192-66c6-4615-a94b-80071566494f/crucible/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.071273542Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"datafile"}
{"msg":"region 911c89d5-f1a1-4a75-b925-3a68a375b2e6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.081504937Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.081664211Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-911c89d5-f1a1-4a75-b925-3a68a375b2e6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.082866153Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.083603323Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"uri":"/crucible/0/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","method":"DELETE","req_id":"e9217307-b9a9-4de0-90e4-d2b3ab6a73ac","remote_addr":"[fd00:1122:3344:101::a]:52257","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23554,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.08685171Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"uri":"/crucible/0/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","method":"GET","req_id":"0f0991d6-0e53-4818-b591-038e94da6384","remote_addr":"[fd00:1122:3344:101::a]:52257","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.087413475Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_db2b1192-66c6-4615-a94b-80071566494f/crucible/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.097553998Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"region":"911c89d5-f1a1-4a75-b925-3a68a375b2e6","component":"worker"}
{"msg":"region 911c89d5-f1a1-4a75-b925-3a68a375b2e6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.159794699Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:10:00.173449535Z","hostname":"oxz_crucible_049e4739-8fda-4544-aa07-31df68ab75c6","pid":22833,"uri":"/crucible/0/regions/911c89d5-f1a1-4a75-b925-3a68a375b2e6","method":"GET","req_id":"1c493fd1-915e-493a-ab05-22f207fe61c3","remote_addr":"[fd00:1122:3344:101::a]:52257","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}