[ Aug 16 07:16:36 Enabled. ]
[ Aug 16 07:16:36 Rereading configuration. ]
[ Aug 16 07:16:38 Rereading configuration. ]
[ Aug 16 07:16:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.037808009Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.038991857Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.039006447Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.064307141Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.26167826Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.88374318Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42733"}
{"msg":"region 99a6dc3a-b434-4617-932a-a330fe6b4ac8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.905376829Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.928654157Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.931795129Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions","method":"POST","req_id":"66fa8734-a331-40f0-8ff3-326b0a3b0858","remote_addr":"[fd00:1122:3344:101::a]:42733","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":33541,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.939575899Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.93960493Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"99a6dc3a-b434-4617-932a-a330fe6b4ac8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.175316304Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"region":"99a6dc3a-b434-4617-932a-a330fe6b4ac8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"99a6dc3a-b434-4617-932a-a330fe6b4ac8\" \"--data\" \"/data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.17732708Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"region":"99a6dc3a-b434-4617-932a-a330fe6b4ac8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.300380532Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions","method":"POST","req_id":"373f69f9-d785-4ca7-ae30-dd6e887ca4ed","remote_addr":"[fd00:1122:3344:101::a]:42733","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":675,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.808660021Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"region":"99a6dc3a-b434-4617-932a-a330fe6b4ac8","component":"worker"}
{"msg":"region 99a6dc3a-b434-4617-932a-a330fe6b4ac8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.808698521Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.808814104Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.809263174Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.846430633Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions","method":"POST","req_id":"ae0f0551-7799-40b2-9558-e0340d29005c","remote_addr":"[fd00:1122:3344:101::a]:42733","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.855856071Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.96232661Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:56.000416739Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:56.014199658Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:56.016169233Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:56.016240135Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:56.016297456Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:56.050015245Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:56.093614972Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.156277734Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48856"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.156450558Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","method":"GET","req_id":"c68b0635-2d65-45d0-b79c-ecd5ca5eed88","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":93,"response_code":"200"}
{"msg":"path is \"/data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.16564256Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"dataset is oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.179395927Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"snapshot is oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8@457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.19294198Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\", \"457023dd-1cbf-42e0-adb9-7dc2029f6007\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.192981901Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.206653467Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8/snapshots/457023dd-1cbf-42e0-adb9-7dc2029f6007","method":"GET","req_id":"4c9e3c83-9d7f-4ed1-b28d-e156bedf6d58","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":45506,"response_code":"200"}
{"msg":"path is \"/data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.215975962Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"dataset is oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.224710564Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"snapshot is oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8@457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.233122758Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\", \"457023dd-1cbf-42e0-adb9-7dc2029f6007\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.233154679Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 99a6dc3a-b434-4617-932a-a330fe6b4ac8-457023dd-1cbf-42e0-adb9-7dc2029f6007 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.241890361Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"applying SMF actions for region 99a6dc3a-b434-4617-932a-a330fe6b4ac8 running snapshot 457023dd-1cbf-42e0-adb9-7dc2029f6007 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.242571897Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.242602617Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8/snapshots/457023dd-1cbf-42e0-adb9-7dc2029f6007/run","method":"POST","req_id":"295131be-3f8b-43a0-89bc-ac7402cbd053","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26663,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246300773Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246385785Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246465636Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246472377Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246516268Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246564169Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246661271Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246723502Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246774414Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.246823295Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.257392429Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-99a6dc3a-b434-4617-932a-a330fe6b4ac8-457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.257846679Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-99a6dc3a-b434-4617-932a-a330fe6b4ac8-457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.261524224Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-99a6dc3a-b434-4617-932a-a330fe6b4ac8-457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.271758511Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8/.zfs/snapshot/457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.271952555Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.272002006Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.272042867Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.272080988Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.272127729Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.275984178Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.280742878Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"region 99a6dc3a-b434-4617-932a-a330fe6b4ac8 running snapshot 457023dd-1cbf-42e0-adb9-7dc2029f6007 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.280758779Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"path is \"/data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.521249722Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"dataset is oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.530139928Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"snapshot is oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8@457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.537872626Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8\", \"457023dd-1cbf-42e0-adb9-7dc2029f6007\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.537907877Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.546325561Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8/snapshots/457023dd-1cbf-42e0-adb9-7dc2029f6007/run","method":"POST","req_id":"5053909b-d818-48f7-9a83-7fc64aa51df8","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":25122,"response_code":"200"}
{"msg":"region ac00c323-f0ca-438f-9baf-8489354aa0a4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.156489698Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.156659292Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions","method":"POST","req_id":"37fe8fcf-f036-4b74-923a-34a6cb94e818","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":230,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.156687632Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.166310135Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.166337395Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ac00c323-f0ca-438f-9baf-8489354aa0a4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.220192939Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"region":"ac00c323-f0ca-438f-9baf-8489354aa0a4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ac00c323-f0ca-438f-9baf-8489354aa0a4\" \"--data\" \"/data/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.222169445Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"region":"ac00c323-f0ca-438f-9baf-8489354aa0a4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.296577073Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"region":"ac00c323-f0ca-438f-9baf-8489354aa0a4","component":"worker"}
{"msg":"region ac00c323-f0ca-438f-9baf-8489354aa0a4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.296614634Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.296718166Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.29729329Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.297371171Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.297452243Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.297464014Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.297502105Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.297552236Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.297932304Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.297996666Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.298047477Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.298094918Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.303656927Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ac00c323-f0ca-438f-9baf-8489354aa0a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.304122587Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ac00c323-f0ca-438f-9baf-8489354aa0a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.31029628Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.319471982Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ac00c323-f0ca-438f-9baf-8489354aa0a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.32241505Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.322649565Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.322705296Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.322753678Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.322780718Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.326445003Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.332849211Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:36.380639594Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions","method":"POST","req_id":"557b13e7-80fd-441b-9b13-cf9b25651077","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.726037724Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","method":"GET","req_id":"dd48a728-9100-4820-90cf-ba74b8e1dfc8","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"path is \"/data/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.728773257Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"dataset is oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.741131293Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"region ac00c323-f0ca-438f-9baf-8489354aa0a4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.751144234Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.751343288Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.75140624Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","method":"DELETE","req_id":"e66df66e-e213-458e-8c12-2f83980382e6","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22661,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-ac00c323-f0ca-438f-9baf-8489354aa0a4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.752460624Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.754991913Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","method":"GET","req_id":"ddd868a2-db4c-45bc-b023-65c8deda6a5c","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":512,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.757363387Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.75745387Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.757544182Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.757599863Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.757605823Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.757632094Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/99a6dc3a-b434-4617-932a-a330fe6b4ac8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.757948071Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.758022963Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.758077784Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.758129375Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.762135478Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.763436798Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0fbcf4bc-94e6-4eb3-8827-d39f09d41305/crucible/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.772976788Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"region":"ac00c323-f0ca-438f-9baf-8489354aa0a4","component":"worker"}
{"msg":"region ac00c323-f0ca-438f-9baf-8489354aa0a4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.820483885Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:24:01.839662268Z","hostname":"oxz_crucible_0537c508-8815-4732-8d29-6bbf60eed6bf","pid":25544,"uri":"/crucible/0/regions/ac00c323-f0ca-438f-9baf-8489354aa0a4","method":"GET","req_id":"c316e814-14b7-4b01-8c96-311b079e9269","remote_addr":"[fd00:1122:3344:101::a]:48856","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":79,"response_code":"200"}