[ Feb 22 01:46:51 Enabled. ]
[ Feb 22 01:46:51 Rereading configuration. ]
[ Feb 22 01:46:52 Rereading configuration. ]
[ Feb 22 01:46:56 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:46:56.189596296Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:46:56.19345286Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:46:56.19346484Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:46:56.214245785Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:46:56.312929715Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.299053213Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:52842"}
{"msg":"region 58ba5051-677c-4041-b886-ee9ecfde672c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.32239657Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.325209279Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions","method":"POST","req_id":"e4c18b46-7bc3-41ef-b612-28c5e76e78d9","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14296,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.325232149Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/58ba5051-677c-4041-b886-ee9ecfde672c","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.335905239Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/58ba5051-677c-4041-b886-ee9ecfde672c","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.33593973Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"58ba5051-677c-4041-b886-ee9ecfde672c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/58ba5051-677c-4041-b886-ee9ecfde672c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.385832743Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"58ba5051-677c-4041-b886-ee9ecfde672c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"58ba5051-677c-4041-b886-ee9ecfde672c\" \"--data\" \"/data/regions/58ba5051-677c-4041-b886-ee9ecfde672c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.387852732Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"58ba5051-677c-4041-b886-ee9ecfde672c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.661152458Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"58ba5051-677c-4041-b886-ee9ecfde672c","component":"worker"}
{"msg":"region 58ba5051-677c-4041-b886-ee9ecfde672c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.66121108Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.661289192Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-58ba5051-677c-4041-b886-ee9ecfde672c","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.661704042Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-58ba5051-677c-4041-b886-ee9ecfde672c","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.669544593Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.670376733Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions","method":"POST","req_id":"dfbeb0b3-5da7-4ea3-adb0-e0a291f589b7","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.68466159Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-58ba5051-677c-4041-b886-ee9ecfde672c","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.690669286Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/58ba5051-677c-4041-b886-ee9ecfde672c","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.691441305Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.693918906Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.693977537Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.694018948Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.711862572Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:37.716119785Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.514286596Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/58ba5051-677c-4041-b886-ee9ecfde672c","method":"GET","req_id":"052aea79-e057-4226-9cc1-b3e3fce5c52b","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"path is \"/data/regions/58ba5051-677c-4041-b886-ee9ecfde672c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.517026393Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"dataset is oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/58ba5051-677c-4041-b886-ee9ecfde672c","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.528345728Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"region 58ba5051-677c-4041-b886-ee9ecfde672c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.538320381Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.538469024Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-58ba5051-677c-4041-b886-ee9ecfde672c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.539649573Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.540801391Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/58ba5051-677c-4041-b886-ee9ecfde672c","method":"DELETE","req_id":"803cd88e-f083-4a35-b575-6120029ed802","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23790,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.544203504Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/58ba5051-677c-4041-b886-ee9ecfde672c","method":"GET","req_id":"feab43f4-7e97-4307-bf32-d0346eaf7f88","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.545090675Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/58ba5051-677c-4041-b886-ee9ecfde672c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.557195749Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"58ba5051-677c-4041-b886-ee9ecfde672c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.617851915Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/58ba5051-677c-4041-b886-ee9ecfde672c","method":"GET","req_id":"6e7d4dbd-1aaa-4170-9df0-a61603df7c5b","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"region 58ba5051-677c-4041-b886-ee9ecfde672c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.626654539Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:38.764488581Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/58ba5051-677c-4041-b886-ee9ecfde672c","method":"GET","req_id":"168ad5c3-922a-499e-881e-5fd258e394ae","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":44,"response_code":"200"}
{"msg":"region df36c583-269b-4c08-b1a7-978f2859157a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.773875597Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.776190523Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.776216534Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions","method":"POST","req_id":"984cc4ad-8669-49f1-9069-7f78944651c8","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2372,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.786305409Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.78633029Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"df36c583-269b-4c08-b1a7-978f2859157a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/df36c583-269b-4c08-b1a7-978f2859157a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.848115022Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"df36c583-269b-4c08-b1a7-978f2859157a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"df36c583-269b-4c08-b1a7-978f2859157a\" \"--data\" \"/data/regions/df36c583-269b-4c08-b1a7-978f2859157a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.848170314Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"df36c583-269b-4c08-b1a7-978f2859157a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.961190692Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"df36c583-269b-4c08-b1a7-978f2859157a","component":"worker"}
{"msg":"region df36c583-269b-4c08-b1a7-978f2859157a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.961240263Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.961319615Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.962670918Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.967205648Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.983969986Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.992049982Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.992335379Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.992398611Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.992444812Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.992486753Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:40.999082184Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:41.005124891Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:47:41.03757016Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions","method":"POST","req_id":"b33c0ac9-b81b-43be-a38c-ed5497bbd407","remote_addr":"[fd00:1122:3344:101::b]:52842","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.170038223Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:50043"}
{"msg":"region 5a941db4-791c-46d3-b759-5c2e62c85815 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.17029756Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.170913144Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions","method":"POST","req_id":"26a1396f-2709-410a-9708-37d7289b732c","remote_addr":"[fd00:1122:3344:101::b]:50043","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":680,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.170929685Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/5a941db4-791c-46d3-b759-5c2e62c85815","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.180570989Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/5a941db4-791c-46d3-b759-5c2e62c85815","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.18061285Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5a941db4-791c-46d3-b759-5c2e62c85815\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5a941db4-791c-46d3-b759-5c2e62c85815\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.254498317Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"5a941db4-791c-46d3-b759-5c2e62c85815","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5a941db4-791c-46d3-b759-5c2e62c85815\" \"--data\" \"/data/regions/5a941db4-791c-46d3-b759-5c2e62c85815\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.256839144Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"5a941db4-791c-46d3-b759-5c2e62c85815","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.400863677Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"region":"5a941db4-791c-46d3-b759-5c2e62c85815","component":"worker"}
{"msg":"region 5a941db4-791c-46d3-b759-5c2e62c85815 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.400929068Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.401080332Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5a941db4-791c-46d3-b759-5c2e62c85815","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.402521847Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5a941db4-791c-46d3-b759-5c2e62c85815","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.417811099Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.437698383Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5a941db4-791c-46d3-b759-5c2e62c85815","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.442255463Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5a941db4-791c-46d3-b759-5c2e62c85815","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.442571651Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.442640603Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.442682064Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.442722405Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.448916945Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.462942676Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:36.533464311Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions","method":"POST","req_id":"708bf851-c737-46e2-95fb-73d94c4c7a4c","remote_addr":"[fd00:1122:3344:101::b]:50043","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.489834449Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/df36c583-269b-4c08-b1a7-978f2859157a","method":"GET","req_id":"151a90d2-a844-4f91-adb7-fa74d52cf0a3","remote_addr":"[fd00:1122:3344:101::b]:50043","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":597,"response_code":"200"}
{"msg":"path is \"/data/regions/df36c583-269b-4c08-b1a7-978f2859157a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.491955781Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"dataset is oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.501889372Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"snapshot is oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a@c128378a-f9db-4e1b-a144-6c99137592dd","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.510059661Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a\", \"c128378a-f9db-4e1b-a144-6c99137592dd\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.510096332Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.519316996Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/df36c583-269b-4c08-b1a7-978f2859157a/snapshots/c128378a-f9db-4e1b-a144-6c99137592dd","method":"GET","req_id":"03bc7177-1493-43cc-bbb2-ed4c324a71d5","remote_addr":"[fd00:1122:3344:101::b]:50043","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27385,"response_code":"200"}
{"msg":"path is \"/data/regions/df36c583-269b-4c08-b1a7-978f2859157a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.522136615Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"dataset is oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.530817336Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"snapshot is oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a@c128378a-f9db-4e1b-a144-6c99137592dd","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.538650646Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a\", \"c128378a-f9db-4e1b-a144-6c99137592dd\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.538684427Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot df36c583-269b-4c08-b1a7-978f2859157a-c128378a-f9db-4e1b-a144-6c99137592dd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.547988113Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"applying SMF actions for region df36c583-269b-4c08-b1a7-978f2859157a running snapshot c128378a-f9db-4e1b-a144-6c99137592dd (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.548173098Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.548193829Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/df36c583-269b-4c08-b1a7-978f2859157a/snapshots/c128378a-f9db-4e1b-a144-6c99137592dd/run","method":"POST","req_id":"6b0a2583-ad3b-4a27-abe6-07693e6c5eda","remote_addr":"[fd00:1122:3344:101::b]:50043","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26084,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-df36c583-269b-4c08-b1a7-978f2859157a-c128378a-f9db-4e1b-a144-6c99137592dd","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.550890764Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-df36c583-269b-4c08-b1a7-978f2859157a-c128378a-f9db-4e1b-a144-6c99137592dd","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.556794218Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-df36c583-269b-4c08-b1a7-978f2859157a-c128378a-f9db-4e1b-a144-6c99137592dd","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.57372379Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/df36c583-269b-4c08-b1a7-978f2859157a/.zfs/snapshot/c128378a-f9db-4e1b-a144-6c99137592dd","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.574052307Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.574105829Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.57415917Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.574207091Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.574266533Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.584365258Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.596476373Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"worker"}
{"msg":"region df36c583-269b-4c08-b1a7-978f2859157a running snapshot c128378a-f9db-4e1b-a144-6c99137592dd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.596499513Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"path is \"/data/regions/df36c583-269b-4c08-b1a7-978f2859157a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.843229393Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"dataset is oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.851954146Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"datafile"}
{"msg":"snapshot is oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a@c128378a-f9db-4e1b-a144-6c99137592dd","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.860476813Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ab346549-5ce3-4d03-94c3-5473cb6f4586/crucible/regions/df36c583-269b-4c08-b1a7-978f2859157a\", \"c128378a-f9db-4e1b-a144-6c99137592dd\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.860511084Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-22T01:58:37.869506793Z","hostname":"oxz_crucible_0e6c70ea-6076-4f47-89ec-29391be27f10","pid":22554,"uri":"/crucible/0/regions/df36c583-269b-4c08-b1a7-978f2859157a/snapshots/c128378a-f9db-4e1b-a144-6c99137592dd/run","method":"POST","req_id":"ffac0d9c-cbd0-4f1c-8986-8310ccc776e5","remote_addr":"[fd00:1122:3344:101::b]:50043","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26307,"response_code":"200"}