微信网站建设咨询上海关键词排名优化怎样
我的需求是在处理几百上千万数据时避免缓存穿透以及缓存击穿情况出现，并且确保数据库和redis同步，为了方便我查询数据操作加了一些条件精准查询和模糊查询以及全字段模糊查询、分页、排序一些小玩意，redis存储是hash表key值也就是数据ID，name值是数据表名和redis同步的，别问为什么，我懒！！
使用示例：
params := utils.QueryParams{Name: "users",Limit: 10,Order: "id",Sort: 1,Where: map[string]interface{}{"name": "张", // 将进行模糊查询"age": 18, // 将进行精确匹配"*": "李", // 将进行全字段模糊查询},
}
results, err := utils.CustomRedisQuery(db, redisClient, params)
if err != nil {// 处理错误
}result, err := GetRedisById(rdb, "users", 1)result, err := GetRedisByWhere(rdb, "users", map[string]interface{}{"status": 1, "type": "vip"}, 1)err = DeleteRedisById(rdb, "users", 1)err = UpdateRedisById(db, rdb, "users", 1)完整代码
/*
+--------------------------------------------------------------------------------
| If this code works, it was written by Xven. If not, I don't know who wrote it.
+--------------------------------------------------------------------------------
| Statement: An Ordinary Person
+--------------------------------------------------------------------------------
| Author: Xven <QQ:270988107>
+--------------------------------------------------------------------------------
| Copyright (c) 2024 Xven All rights reserved.
+--------------------------------------------------------------------------------
*/
package utilsimport ("context""encoding/json""fmt""sort""strings""sync""time""github.com/go-redis/redis/v8""gorm.io/gorm"
)type QueryParams struct {Name string // 表名Limit int // 分页数量Order string // 排序字段Sort int // 排序方式 1:升序 2:降序Where map[string]interface{} // 查询条件
}/*** 检查字符串是否包含子串* @Author Xven <270988107@qq.com>* @param {string} str* @param {string} substr* @return {bool}*/
func containsString(str, substr string) bool {return strings.Contains(strings.ToLower(str), strings.ToLower(substr))
}/*** 检查字符串是否包含通配符* @Author Xven <270988107@qq.com>* @param {string} str* @return {bool}*/
func hasWildcard(str string) bool {return strings.Contains(str, "*")
}/*** 对数据进行排序* @Author Xven <270988107@qq.com>* @param {[]map[string]interface{}} data* @param {string} orderField* @param {int} sortType* @return {void}*/
func sortData(data []map[string]interface{}, orderField string, sortType int) {sort.Slice(data, func(i, j int) bool {if sortType == 1 { // 升序return fmt.Sprint(data[i][orderField]) < fmt.Sprint(data[j][orderField])}return fmt.Sprint(data[i][orderField]) > fmt.Sprint(data[j][orderField])})
}/*** 自定义redis查询* @Author Xven <270988107@qq.com>* @param {*gorm.DB} db* @param {*redis.Client} rdb* @param {QueryParams} params* @return {[]map[string]interface{}, error}*/
func CustomRedisQuery(db *gorm.DB, rdb *redis.Client, params QueryParams) ([]map[string]interface{}, error) {ctx := context.Background()var result []map[string]interface{}// 参数校验,防止缓存穿透if params.Name == "" {return nil, fmt.Errorf("表名不能为空")}// 构建 Redis keyredisKey := params.Name + ":list"// 使用分布式锁防止缓存击穿lockKey := fmt.Sprintf("lock:%s", redisKey)lock := rdb.SetNX(ctx, lockKey, "1", 10*time.Second)if !lock.Val() {// 等待100ms后重试time.Sleep(100 * time.Millisecond)return CustomRedisQuery(db, rdb, params)}defer rdb.Del(ctx, lockKey)// 1. 先查询 Redis 缓存vals, err := rdb.HGetAll(ctx, redisKey).Result()if err == nil && len(vals) > 0 {// 将缓存数据解析为结果集for _, v := range vals {var item map[string]interface{}if err := json.Unmarshal([]byte(v), &item); err == nil {result = append(result, item)}}// 如果有查询条件,则进行过滤if len(params.Where) > 0 {result = filterData(result, params.Where)}// 处理排序if params.Order != "" {sortData(result, params.Order, params.Sort)}// 处理分页if params.Limit > 0 && len(result) > params.Limit {result = result[:params.Limit]}return result, nil}// 2. Redis 没有数据,从数据库查询var dbResult []map[string]interface{}// 使用连接池控制并发pool := make(chan struct{}, 10)var wg sync.WaitGroupvar mu sync.Mutex// 使用游标分批查询数据库,避免一次性加载过多数据err = db.Table(params.Name).FindInBatches(&dbResult, 1000, func(tx *gorm.DB, batch int) error {wg.Add(1)pool <- struct{}{} // 获取连接go func(data []map[string]interface{}) {defer func() {<-pool // 释放连接wg.Done()}()pipe := rdb.Pipeline()// 批量写入Redisfor _, item := range data {// 将每条记录序列化为JSONjsonData, err := json.Marshal(item)if err != nil {continue}// 使用ID作为field,JSON作为value写入hashid := fmt.Sprint(item["id"])pipe.HSet(ctx, redisKey, id, string(jsonData))}// 执行管道命令_, err := pipe.Exec(ctx)if err != nil {// 写入失败时重试写入数据for _, item := range data {jsonData, _ := json.Marshal(item)id := fmt.Sprint(item["id"])rdb.HSet(ctx, redisKey, id, string(jsonData))}}mu.Lock()result = append(result, data...)mu.Unlock()}(dbResult)return nil}).Errorwg.Wait()if err != nil {// 设置空值缓存,防止缓存穿透rdb.Set(ctx, redisKey+"_empty", "1", 5*time.Minute)return nil, err}// 处理排序if params.Order != "" {sortData(result, params.Order, params.Sort)}// 处理分页if params.Limit > 0 && len(result) > params.Limit {result = result[:params.Limit]}return result, nil
}/*** 过滤数据* @Author Xven <270988107@qq.com>* @param {[]map[string]interface{}} data* @param {map[string]interface{}} where* @return {[]map[string]interface{}}*/
func filterData(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {var filteredResult []map[string]interface{}// 先处理精确匹配条件hasExactMatch := falsefor field, value := range where {if field != "*" {if strValue, ok := value.(string); ok && !hasWildcard(strValue) {hasExactMatch = truebreak} else if !ok {hasExactMatch = truebreak}}}if hasExactMatch {filteredResult = exactMatch(data, where)if len(filteredResult) > 0 {filteredResult = fuzzyMatch(filteredResult, where)}} else {filteredResult = fuzzyMatch(data, where)}return filteredResult
}/*** 精确匹配* @Author Xven <270988107@qq.com>* @param {[]map[string]interface{}} data* @param {map[string]interface{}} where* @return {[]map[string]interface{}}*/
func exactMatch(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {var result []map[string]interface{}for _, item := range data {matched := truefor field, value := range where {if field == "*" {continue}if strValue, ok := value.(string); ok {if !hasWildcard(strValue) {if itemValue, exists := item[field]; !exists || itemValue != value {matched = falsebreak}}} else {if itemValue, exists := item[field]; !exists || itemValue != value {matched = falsebreak}}}if matched {result = append(result, item)}}return result
}/*** 模糊匹配* @Author Xven <270988107@qq.com>* @param {[]map[string]interface{}} data* @param {map[string]interface{}} where* @return {[]map[string]interface{}}*/
func fuzzyMatch(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {var result []map[string]interface{}// 处理指定字段的模糊查询for _, item := range data {matched := truefor field, value := range where {if field == "*" {continue}if strValue, ok := value.(string); ok && hasWildcard(strValue) {if itemValue, exists := item[field]; exists {if strItemValue, ok := itemValue.(string); ok {pattern := strings.ReplaceAll(strValue, "*", "")if !strings.Contains(strings.ToLower(strItemValue), strings.ToLower(pattern)) {matched = falsebreak}}}}}if matched {result = append(result, item)}}// 处理全字段模糊查询if wildcardValue, exists := where["*"]; exists {var globalResult []map[string]interface{}searchData := resultif len(searchData) == 0 {searchData = data}if strValue, ok := wildcardValue.(string); ok {for _, item := range searchData {matched := falsefor _, fieldValue := range item {if strFieldValue, ok := fieldValue.(string); ok {if containsString(strFieldValue, strValue) {matched = truebreak}}}if matched {globalResult = append(globalResult, item)}}}result = globalResult}return result
}/*** 根据ID查询单条数据* @Author Xven <270988107@qq.com>* @param {*redis.Client} rdb* @param {string} name* @param {interface{}} id* @return {map[string]interface{}, error}*/
func GetRedisById(rdb *redis.Client, name string, id interface{}) (map[string]interface{}, error) {ctx := context.Background()redisKey := name + ":list"// 从Redis查询jsonData, err := rdb.HGet(ctx, redisKey, fmt.Sprint(id)).Result()if err == nil {// Redis命中,解析JSON数据var result map[string]interface{}err = json.Unmarshal([]byte(jsonData), &result)if err == nil {return result, nil}}return nil, err
}/*** 根据条件查询数据* @Author Xven <270988107@qq.com>* @param {*redis.Client} rdb* @param {string} name* @param {map[string]interface{}} where* @param {int} is* @return {interface{}, error}*/
func GetRedisByWhere(rdb *redis.Client, name string, where map[string]interface{}, is int) (interface{}, error) {ctx := context.Background()redisKey := name + ":list"// 获取所有数据values, err := rdb.HGetAll(ctx, redisKey).Result()if err != nil {return nil, err}var results []map[string]interface{}// 遍历所有数据进行条件匹配for _, jsonStr := range values {var item map[string]interface{}err := json.Unmarshal([]byte(jsonStr), &item)if err != nil {continue}// 检查是否匹配所有条件match := truefor k, v := range where {if item[k] != v {match = falsebreak}}if match {results = append(results, item)// 如果是单条查询且已找到,则直接返回if is == 0 {return item, nil}}}if is == 0 {return nil, nil}return results, nil
}/*** 删除指定ID的数据* @Author Xven <270988107@qq.com>* @param {*redis.Client} rdb* @param {string} name* @param {interface{}} id* @return {error}*/
func DeleteRedisById(rdb *redis.Client, name string, id interface{}) error {ctx := context.Background()redisKey := name + ":list"maxRetries := 3for i := 0; i < maxRetries; i++ {err := rdb.HDel(ctx, redisKey, fmt.Sprint(id)).Err()if err == nil {return nil}// 重试前等待短暂时间time.Sleep(time.Millisecond * 100)}return fmt.Errorf("failed to delete after %d retries", maxRetries)
}/*** 更新指定ID的数据* @Author Xven <270988107@qq.com>* @param {*gorm.DB} db* @param {*redis.Client} rdb* @param {string} name* @param {interface{}} id* @return {error}*/
func UpdateRedisById(db *gorm.DB, rdb *redis.Client, name string, id interface{}) error {ctx := context.Background()redisKey := name + ":list"maxRetries := 3// 从数据库查询数据var result map[string]interface{}err := db.Table(name).Where("id = ?", id).Take(&result).Errorif err != nil {return err}// 序列化数据jsonData, err := json.Marshal(result)if err != nil {return err}// 重试更新Redisfor i := 0; i < maxRetries; i++ {err = rdb.HSet(ctx, redisKey, fmt.Sprint(id), string(jsonData)).Err()if err == nil {return nil}time.Sleep(time.Millisecond * 100)}return fmt.Errorf("failed to update after %d retries", maxRetries)
}