[ Feb 28 10:02:22 Enabled. ]
[ Feb 28 10:02:22 Rereading configuration. ]
[ Feb 28 10:02:25 Rereading configuration. ]
[ Feb 28 10:02:33 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:33.266653122Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:33.269623951Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:33.269633491Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:33.284933711Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:33.556753267Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.006881945Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34962"}
{"msg":"region d87b249c-8f2a-45d5-aec9-e928d13a1b07 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.012057173Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.012970545Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions","method":"POST","req_id":"80693fca-c5fd-4062-985f-2fe048638daa","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":1633,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.012984605Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.022884184Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.022940825Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d87b249c-8f2a-45d5-aec9-e928d13a1b07\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.108860305Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"region":"d87b249c-8f2a-45d5-aec9-e928d13a1b07","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d87b249c-8f2a-45d5-aec9-e928d13a1b07\" \"--data\" \"/data/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.108966247Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"region":"d87b249c-8f2a-45d5-aec9-e928d13a1b07","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.271992674Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"region":"d87b249c-8f2a-45d5-aec9-e928d13a1b07","component":"worker"}
{"msg":"region d87b249c-8f2a-45d5-aec9-e928d13a1b07 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.272069905Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.272270737Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d87b249c-8f2a-45d5-aec9-e928d13a1b07","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.272664802Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d87b249c-8f2a-45d5-aec9-e928d13a1b07","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.282300458Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.299216129Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d87b249c-8f2a-45d5-aec9-e928d13a1b07","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.30926819Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.310149691Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.310237823Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.310292963Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.310335814Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.321448089Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.331253537Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:49.378161599Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions","method":"POST","req_id":"c8411649-ad3c-410f-adbb-7630433d391d","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.261647054Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","method":"GET","req_id":"8c08e602-2438-4c50-8bea-0b775a529adf","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2233,"response_code":"200"}
{"msg":"path is \"/data/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.265779188Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"dataset is oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.27973511Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"region d87b249c-8f2a-45d5-aec9-e928d13a1b07 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.289830991Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.290053114Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.290111965Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","method":"DELETE","req_id":"bb90111d-1bd6-49b6-9706-c8c4dfa98d88","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24367,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-d87b249c-8f2a-45d5-aec9-e928d13a1b07 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.291070807Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.294100447Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","method":"GET","req_id":"43313af5-e404-4f27-b2d4-c7fffe01f5d2","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.301036877Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.312289854Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"region":"d87b249c-8f2a-45d5-aec9-e928d13a1b07","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.386626524Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","method":"GET","req_id":"7c1e6f70-ab91-45fd-9667-a99690cfd05a","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region d87b249c-8f2a-45d5-aec9-e928d13a1b07 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.397212072Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:50.495769868Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/d87b249c-8f2a-45d5-aec9-e928d13a1b07","method":"GET","req_id":"f7f1a071-dacb-45da-995c-91e12333626f","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":43,"response_code":"200"}
{"msg":"region 5f7086d1-69d0-4025-a97a-a5fb69e67fda state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.605234206Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.605419008Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.605497149Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions","method":"POST","req_id":"8bea22c8-b5fe-4f3d-9594-3a21f6ec2cd5","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":257,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.614395695Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.614424056Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5f7086d1-69d0-4025-a97a-a5fb69e67fda\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.688516122Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"region":"5f7086d1-69d0-4025-a97a-a5fb69e67fda","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5f7086d1-69d0-4025-a97a-a5fb69e67fda\" \"--data\" \"/data/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.690481458Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"region":"5f7086d1-69d0-4025-a97a-a5fb69e67fda","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.766035104Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"region":"5f7086d1-69d0-4025-a97a-a5fb69e67fda","component":"worker"}
{"msg":"region 5f7086d1-69d0-4025-a97a-a5fb69e67fda state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.766099394Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.766315777Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.767829767Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.771614506Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.779736222Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.78334729Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.783647503Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.783710084Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.783748235Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.783791875Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.788125372Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.788239213Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions","method":"POST","req_id":"8e748647-2f34-4c69-b2e4-d5995c69ac50","remote_addr":"[fd00:1122:3344:101::a]:34962","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:02:52.793094476Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.751998434Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60863"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.752126586Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda","method":"GET","req_id":"e9eab859-58db-44e4-a0c3-24f08613bd0f","remote_addr":"[fd00:1122:3344:101::a]:60863","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"path is \"/data/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.758970635Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"dataset is oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.768359398Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"snapshot is oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda@7530d1d3-6aab-42d0-b33d-cab3169b20bf","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.777565568Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\", \"7530d1d3-6aab-42d0-b33d-cab3169b20bf\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.777594858Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.788805344Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda/snapshots/7530d1d3-6aab-42d0-b33d-cab3169b20bf","method":"GET","req_id":"b970d6e6-b375-4cb3-a650-16c6c665824d","remote_addr":"[fd00:1122:3344:101::a]:60863","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":33242,"response_code":"200"}
{"msg":"path is \"/data/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.792343631Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"dataset is oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.800992282Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"snapshot is oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda@7530d1d3-6aab-42d0-b33d-cab3169b20bf","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.809130839Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\", \"7530d1d3-6aab-42d0-b33d-cab3169b20bf\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.809159729Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 5f7086d1-69d0-4025-a97a-a5fb69e67fda-7530d1d3-6aab-42d0-b33d-cab3169b20bf state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.818727844Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"applying SMF actions for region 5f7086d1-69d0-4025-a97a-a5fb69e67fda running snapshot 7530d1d3-6aab-42d0-b33d-cab3169b20bf (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.819272431Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.819294371Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda/snapshots/7530d1d3-6aab-42d0-b33d-cab3169b20bf/run","method":"POST","req_id":"cc801279-a1c9-485c-a283-de7ed9332b41","remote_addr":"[fd00:1122:3344:101::a]:60863","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27000,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-5f7086d1-69d0-4025-a97a-a5fb69e67fda-7530d1d3-6aab-42d0-b33d-cab3169b20bf","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.826195251Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5f7086d1-69d0-4025-a97a-a5fb69e67fda-7530d1d3-6aab-42d0-b33d-cab3169b20bf","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.832674376Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5f7086d1-69d0-4025-a97a-a5fb69e67fda-7530d1d3-6aab-42d0-b33d-cab3169b20bf","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.844456609Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda/.zfs/snapshot/7530d1d3-6aab-42d0-b33d-cab3169b20bf","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.844682283Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.844732733Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.844772844Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.844810574Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.844855165Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.848814986Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.853250334Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"worker"}
{"msg":"region 5f7086d1-69d0-4025-a97a-a5fb69e67fda running snapshot 7530d1d3-6aab-42d0-b33d-cab3169b20bf state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.853265575Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"path is \"/data/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.956465171Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"dataset is oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.965030982Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"datafile"}
{"msg":"snapshot is oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda@7530d1d3-6aab-42d0-b33d-cab3169b20bf","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.972629602Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_de578b84-0a5b-48c2-9460-d67ca9c52e91/crucible/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda\", \"7530d1d3-6aab-42d0-b33d-cab3169b20bf\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.972657172Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T10:13:31.980019478Z","hostname":"oxz_crucible_d133f7d4-be2d-4d6c-8518-49e6334a3cef","pid":24671,"uri":"/crucible/0/regions/5f7086d1-69d0-4025-a97a-a5fb69e67fda/snapshots/7530d1d3-6aab-42d0-b33d-cab3169b20bf/run","method":"POST","req_id":"105820ef-7c17-43a9-a064-4f0397e40999","remote_addr":"[fd00:1122:3344:101::a]:60863","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":23571,"response_code":"200"}