[ Mar 11 22:06:53 Enabled. ]
[ Mar 11 22:06:53 Rereading configuration. ]
[ Mar 11 22:06:55 Rereading configuration. ]
[ Mar 11 22:07:02 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_76257370-c614-451e-84b5-c6ea07ca98ba/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_76257370-c614-451e-84b5-c6ea07ca98ba/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:02.663883005Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:02.666549868Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:02.666558688Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:02.678705312Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:02.923220626Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:55.962955618Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37623"}
{"msg":"region c70a6c99-9ccd-44fb-a209-bd54b56b5c0e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.000929662Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.003671496Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions","method":"POST","req_id":"1d5aca93-4a7d-4080-adaf-31db70e68638","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":17381,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.00420143Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_76257370-c614-451e-84b5-c6ea07ca98ba/crucible/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.014131335Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_76257370-c614-451e-84b5-c6ea07ca98ba/crucible/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.014169975Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c70a6c99-9ccd-44fb-a209-bd54b56b5c0e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.101288547Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"region":"c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c70a6c99-9ccd-44fb-a209-bd54b56b5c0e\" \"--data\" \"/data/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.103253884Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"region":"c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.201069858Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions","method":"POST","req_id":"7479f9e4-f0d8-4729-bc3b-69b4db9c2081","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.28462807Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"region":"c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","component":"worker"}
{"msg":"region c70a6c99-9ccd-44fb-a209-bd54b56b5c0e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.284668911Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.284769851Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.285174875Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.304064026Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.323800724Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.331199727Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.331973574Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.332040144Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.332083945Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.332122755Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.343500762Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.368743067Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.580976456Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions","method":"POST","req_id":"f1ccd9a6-28a2-471a-9683-22df977ec07d","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.318972627Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","method":"GET","req_id":"842328e2-109d-4c34-8f99-c74607c711c9","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":1943,"response_code":"200"}
{"msg":"path is \"/data/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.322352176Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"datafile"}
{"msg":"dataset is oxp_76257370-c614-451e-84b5-c6ea07ca98ba/crucible/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.332733274Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"datafile"}
{"msg":"region c70a6c99-9ccd-44fb-a209-bd54b56b5c0e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.34288322Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.343020242Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.343065002Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","method":"DELETE","req_id":"c87d2bb1-a1a8-4ba6-aee3-13025e49eb7e","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20752,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-c70a6c99-9ccd-44fb-a209-bd54b56b5c0e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.344077121Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.346683143Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","method":"GET","req_id":"78496c37-7e6b-4708-b51b-d25b862c2eeb","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.358459823Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_76257370-c614-451e-84b5-c6ea07ca98ba/crucible/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.370369145Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"region":"c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.455613421Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","method":"GET","req_id":"f67bf98c-218a-4e54-9020-33fbe4c1c78c","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"region c70a6c99-9ccd-44fb-a209-bd54b56b5c0e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.478575257Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.562947666Z","hostname":"oxz_crucible_3864b56a-ead4-4981-8e98-d5c420b44e60","pid":22692,"uri":"/crucible/0/regions/c70a6c99-9ccd-44fb-a209-bd54b56b5c0e","method":"GET","req_id":"376c9f64-92e0-4dc4-9f7f-2cd105aa523a","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":34,"response_code":"200"}