GELU

GELU

这个函数特别占内存,计算量很大，对检测有帮助,收敛比relu6快

但是最高精度没有relu6高

梯度最大在第一层卷积层

类：

  

   

    

     

    
    

     

      #!/usr/bin/env python3
     
    
   

    

     

    
    

     

      # -*- coding: utf-8 -*-
     
    
   

    

     

    
    

     

      import torch
     
    
   

    

     

    
    

     

      import math
     
    
   

    

     

    
    

     

      from torch import nn
     
    
   

    

     

    
    

     

      from torch.nn import functional as F
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      class mish(nn.Module):
     
    
   

    

     

    
    

     
 def __init__(self):
     
    
   

    

     

    
    

     

       super(mish, self).__init__()
     
    
   

    

     

    
    

     
 # Also see https://arxiv.org/abs/1606.08415
     
    
   

    

     

    
    

     
 def forward(self, x):
     
    
   

    

     

    
    

     
 return x * torch.tanh(F.softplus(x))
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      class Gelu(nn.Module):
     
    
   

    

     

    
    

     
 def __init__(self):
     
    
   

    

     

    
    

     

       super(Gelu, self).__init__()
     
    
   

    

     

    
    

     
 # Also see https://arxiv.org/abs/1606.08415
     
    
   

    

     

    
    

     
 def forward(self, x):
     
    
   

    

     

    
    

     
 return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      class Gelu_new(nn.Module):
     
    
   

    

     

    
    

     
 def __init__(self):
     
    
   

    

     

    
    

     

       super(Gelu_new, self).__init__()
     
    
   

    

     

    
    

     
 #Also see https://arxiv.org/abs/1606.08415
     
    
   

    

     

    
    

     
 def forward(self, x):
     
    
   

    

     

    
    

     
 return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 *
     
    
  
 

文章来源: blog.csdn.net，作者：网奇，版权归原作者所有，如需转载，请联系作者。

原文链接：blog.csdn.net/jacke121/article/details/103767214