How to edit or create new smileys in wordpress?

To modify or add the code of the smilies you must set the global $wpsmiliestrans before WordPress. To modify or add the images of the smilies you must add the filter smilies_src as described in the documentation.

This is the type of code that should be placed in a plugin , because when changing themes we want custom smilies to continue working.

<?php
/**
 * Plugin Name: (SOPT) Smilies
 * Plugin Author: brasofilo
 * Plugin URI: http://pt.stackoverflow.com/q/23604/201
 */

/**
 * Substituir a global $wpsmiliestrans usando uma prioridade menor que a padrão do WP (5)
 */
add_action( 'init', 'b5f_smilies_init', 4 );

function my_smilies_init()
{
    global $wpsmiliestrans;

    # Lista padrão do WP, imagens dentro do diretório /wp-includes/images/smilies
    $wpsmiliestrans = array(
        ':mrgreen:' => 'icon_mrgreen.gif',
        ':neutral:' => 'icon_neutral.gif',
        ':twisted:' => 'icon_twisted.gif',
          ':arrow:' => 'icon_arrow.gif',
          ':shock:' => 'icon_eek.gif',
          ':smile:' => 'icon_smile.gif',
            ':???:' => 'icon_confused.gif',
           ':cool:' => 'icon_cool.gif',
           ':evil:' => 'icon_evil.gif',
           ':grin:' => 'icon_biggrin.gif',
           ':idea:' => 'icon_idea.gif',
           ':oops:' => 'icon_redface.gif',
           ':razz:' => 'icon_razz.gif',
           ':roll:' => 'icon_rolleyes.gif',
           ':wink:' => 'icon_wink.gif',
            ':cry:' => 'icon_cry.gif',
            ':eek:' => 'icon_surprised.gif',
            ':lol:' => 'icon_lol.gif',
            ':mad:' => 'icon_mad.gif',
            ':sad:' => 'icon_sad.gif',
              '8-)' => 'icon_cool.gif',
              '8-O' => 'icon_eek.gif',
              ':-(' => 'icon_sad.gif',
              ':-)' => 'icon_smile.gif',
              ':-?' => 'icon_confused.gif',
              ':-D' => 'icon_biggrin.gif',
              ':-P' => 'icon_razz.gif',
              ':-o' => 'icon_surprised.gif',
              ':-x' => 'icon_mad.gif',
              ':-|' => 'icon_neutral.gif',
              ';-)' => 'icon_wink.gif',
        // This one transformation breaks regular text with frequency.
        //     '8)' => 'icon_cool.gif',
               '8O' => 'icon_eek.gif',
               ':(' => 'icon_sad.gif',
               ':)' => 'icon_smile.gif',
               ':?' => 'icon_confused.gif',
               ':D' => 'icon_biggrin.gif',
               ':P' => 'icon_razz.gif',
               ':o' => 'icon_surprised.gif',
               ':x' => 'icon_mad.gif',
               ':|' => 'icon_neutral.gif',
               ';)' => 'icon_wink.gif',
              ':!:' => 'icon_exclaim.gif',
              ':?:' => 'icon_question.gif',
    );
}

/**
 * Substituir a imagem do smiley Icon Cool
 * colocar as imagens dentro de /my-plugin/images/
 */
add_filter( 'smilies_src', 'b5f_smilies_src', 10, 3 );

function b5f_smilies_src( $path, $img, $site_url )
{
    $img_dir = plugins_url( '/images/', __FILE__ );
    if( 'icon_cool.gif' === $img )
        $path = $img_dir . 'icon_cool.gif';

    return $path;
}