[ Oct 9 17:19:22 Enabled. ] [ Oct 9 17:19:23 Rereading configuration. ] [ Oct 9 17:19:25 Rereading configuration. ] [ Oct 9 17:19:27 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:19:27.963156772Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710} {"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:19:27.965107214Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:19:27.965120684Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:19:27.988425796Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:19:28.173685585Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.61102879Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55288"} {"msg":"region 79918c08-9bf3-4d3d-836f-97e14700cefe state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.638981272Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.643399197Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions","method":"POST","req_id":"1b392d7a-2584-4775-9fbe-08387f14acf8","remote_addr":"[fd00:1122:3344:101::a]:55288","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":8492,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.644654784Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.654010876Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.654041597Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"79918c08-9bf3-4d3d-836f-97e14700cefe\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.714947058Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"region":"79918c08-9bf3-4d3d-836f-97e14700cefe","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"79918c08-9bf3-4d3d-836f-97e14700cefe\" \"--data\" \"/data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.718768111Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"region":"79918c08-9bf3-4d3d-836f-97e14700cefe","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.907407993Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions","method":"POST","req_id":"f50cc862-9fc0-4b6d-93ca-611b69bd7c28","remote_addr":"[fd00:1122:3344:101::a]:55288","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":80,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.953751151Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"region":"79918c08-9bf3-4d3d-836f-97e14700cefe","component":"worker"} {"msg":"region 79918c08-9bf3-4d3d-836f-97e14700cefe state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.953786702Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.953895714Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.956148033Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.960796543Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.975492329Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.978188457Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.98525361Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.985315721Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.985361332Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.985400043Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.988694833Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:41.993228331Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:20:42.63588359Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions","method":"POST","req_id":"c7cfe3da-5ada-4446-b6ad-20a4cbb8e59a","remote_addr":"[fd00:1122:3344:101::a]:55288","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.681270281Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49966"} {"msg":"region eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.681445875Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.682353255Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.682380225Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions","method":"POST","req_id":"798d0de2-e98b-433f-a74b-828db465a64d","remote_addr":"[fd00:1122:3344:101::a]:49966","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":958,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.693940334Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.693985505Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.770651886Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"region":"eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f\" \"--data\" \"/data/regions/eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.772927245Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"region":"eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.84139564Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"region":"eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","component":"worker"} {"msg":"region eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.841439681Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.841542943Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.85300373Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853120592Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853241475Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853254495Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853313766Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853394948Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853546772Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853612023Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853675124Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.853763116Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.861043113Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.861643176Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.864893916Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.87299205Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.877777393Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb3ae7e5-a9d4-42f8-8c28-c324c8d20e2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.87808875Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.878176652Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.878230443Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.878277814Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.881903272Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.8864557Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:00.961066667Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions","method":"POST","req_id":"8a2b19e7-712f-48c7-b582-10db49bcec84","remote_addr":"[fd00:1122:3344:101::a]:49966","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":88,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.20941714Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","method":"GET","req_id":"52329613-b6c0-4c14-8bd2-2e047379b6a7","remote_addr":"[fd00:1122:3344:101::a]:49966","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3607,"response_code":"200"} {"msg":"path is \"/data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.212509946Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"dataset is oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.222477311Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"snapshot is oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe@5a769270-2edb-4f6e-8f69-40585f1d8665","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.231986946Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\", \"5a769270-2edb-4f6e-8f69-40585f1d8665\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.232015346Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.247583762Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions/79918c08-9bf3-4d3d-836f-97e14700cefe/snapshots/5a769270-2edb-4f6e-8f69-40585f1d8665","method":"GET","req_id":"d5e06fc0-4d29-497b-b9cb-5e9c0fb4df53","remote_addr":"[fd00:1122:3344:101::a]:49966","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":35068,"response_code":"200"} {"msg":"path is \"/data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.251320462Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"dataset is oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.261479741Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"snapshot is oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe@5a769270-2edb-4f6e-8f69-40585f1d8665","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.269660207Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\", \"5a769270-2edb-4f6e-8f69-40585f1d8665\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.269717569Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 79918c08-9bf3-4d3d-836f-97e14700cefe-5a769270-2edb-4f6e-8f69-40585f1d8665 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.279885457Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"applying SMF actions for region 79918c08-9bf3-4d3d-836f-97e14700cefe running snapshot 5a769270-2edb-4f6e-8f69-40585f1d8665 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.280082842Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.280118042Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions/79918c08-9bf3-4d3d-836f-97e14700cefe/snapshots/5a769270-2edb-4f6e-8f69-40585f1d8665/run","method":"POST","req_id":"92c5b1bb-75b1-45c0-84bf-bf472fe374f5","remote_addr":"[fd00:1122:3344:101::a]:49966","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28820,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.280682545Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.280765266Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.280847418Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.280854748Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.280894999Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.28094009Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.281314728Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.28137696Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.281426591Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.281478512Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.284967287Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-79918c08-9bf3-4d3d-836f-97e14700cefe-5a769270-2edb-4f6e-8f69-40585f1d8665","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.286376947Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-79918c08-9bf3-4d3d-836f-97e14700cefe-5a769270-2edb-4f6e-8f69-40585f1d8665","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.290618668Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-79918c08-9bf3-4d3d-836f-97e14700cefe-5a769270-2edb-4f6e-8f69-40585f1d8665","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.301117545Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe/.zfs/snapshot/5a769270-2edb-4f6e-8f69-40585f1d8665","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.301337459Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.30138872Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.301429561Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.301468642Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.301521533Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.306220735Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.310534557Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"worker"} {"msg":"region 79918c08-9bf3-4d3d-836f-97e14700cefe running snapshot 5a769270-2edb-4f6e-8f69-40585f1d8665 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.310556568Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"path is \"/data/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.54142601Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"dataset is oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.550200609Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"datafile"} {"msg":"snapshot is oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe@5a769270-2edb-4f6e-8f69-40585f1d8665","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.558858085Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5dfbbc08-97d8-40a6-a4c0-1c0a4f79322a/crucible/regions/79918c08-9bf3-4d3d-836f-97e14700cefe\", \"5a769270-2edb-4f6e-8f69-40585f1d8665\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.558887716Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T17:24:02.566245174Z","hostname":"oxz_crucible_e22bd1fb-80a9-4517-a77f-c1d8dcb33aa2","pid":21710,"uri":"/crucible/0/regions/79918c08-9bf3-4d3d-836f-97e14700cefe/snapshots/5a769270-2edb-4f6e-8f69-40585f1d8665/run","method":"POST","req_id":"c26f26d7-1863-47af-8782-234936fcd390","remote_addr":"[fd00:1122:3344:101::a]:49966","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24850,"response_code":"200"}